{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "unable to import 'smart_open.gcs', disabling that module\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import torch\n",
    "import gensim\n",
    "from gensim.test.utils import datapath, get_tmpfile\n",
    "from gensim.models import KeyedVectors\n",
    "from gensim.scripts.glove2word2vec import glove2word2vec\n",
    "from transformers import *\n",
    "from transformers import AdamW\n",
    "import torch.utils.data as Data\n",
    "import collections\n",
    "import os\n",
    "import random\n",
    "import tarfile\n",
    "import torch\n",
    "from torch import nn\n",
    "import torchtext.vocab as Vocab\n",
    "import pickle as pk\n",
    "import copy\n",
    "import time\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "import torch.nn.functional as F\n",
    "from IPython.display import display,HTML\n",
    "import os\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from torch.nn.utils.rnn import pack_padded_sequence\n",
    "from torch.nn.utils.rnn import pad_packed_sequence\n",
    "from torch.nn.utils.rnn import pack_sequence\n",
    "from torch.nn import CrossEntropyLoss, MSELoss\n",
    "from sklearn import metrics\n",
    "import joblib\n",
    "import math\n",
    "import argparse\n",
    "import glob\n",
    "import json\n",
    "import logging\n",
    "import unicodedata\n",
    "from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset\n",
    "from torch.utils.data.distributed import DistributedSampler\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "import torch.utils.data as Data\n",
    "import jieba\n",
    "import jieba.posseg as pseg\n",
    "import argparse\n",
    "logger = logging.getLogger(__name__)\n",
    "logging.basicConfig(\n",
    "    format=\"%(asctime)s - %(levelname)s - %(name)s -   %(message)s\",\n",
    "    datefmt=\"%m/%d/%Y %H:%M:%S\",\n",
    "    level=logging.INFO \n",
    ")\n",
    "ARG=collections.namedtuple('ARG',['train_batch_size',\n",
    " 'eval_batch_size',\n",
    " 'weight_decay',\n",
    " 'learning_rate',\n",
    " 'adam_epsilon',\n",
    " 'num_train_epochs',\n",
    " 'warmup_steps',\n",
    " 'gradient_accumulation_steps',\n",
    " 'save_steps',\n",
    " 'max_grad_norm',\n",
    " 'model_name_or_path',\n",
    " 'output_dir',\n",
    " 'seed',\n",
    " 'device',\n",
    " 'n_gpu',\n",
    " 'max_steps',\n",
    " 'output_mode',\n",
    "'fp16_opt_level',\n",
    "'fp16',\n",
    "'card_list'])\n",
    "# parser = argparse.ArgumentParser()\n",
    "# parser.add_argument(\"-l\",\"--loss_type\", help=\"loss\",type=int,default=1)\n",
    "# parser.add_argument(\"-f\",\"--use_feature\", help=\"feature\",type=int,default=1)\n",
    "# parser.add_argument(\"-c\",\"--card\", help=\"feature\",type=int,default=0)\n",
    "# args = parser.parse_args()\n",
    "# output_dir=\"./output_\"+str(args.loss_type)+\"_\"+str(args.use_feature)+\"/\"\n",
    "# print(output_dir)\n",
    "# loss_type=bool(args.loss_type)\n",
    "# use_feature=bool(args.use_feature)\n",
    "# print(\"loss_type:\",loss_type)\n",
    "# print(\"use_feature:\",use_feature)\n",
    "# print(\"cuda:\"+str(args.card))\n",
    "# device=torch.device(\"cuda:\"+str(args.card) if torch.cuda.is_available() else \"cpu\")\n",
    "# if args.card==0:\n",
    "#     card_list=[0,1]\n",
    "# if args.card==2:\n",
    "#     card_list=[2,3]\n",
    "# print(\"card_list\",card_list) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "code_folding": [
     0,
     16,
     35,
     48,
     58,
     81,
     96,
     114,
     118,
     123,
     129,
     131
    ]
   },
   "outputs": [],
   "source": [
    "def convert_text_to_ids(tokenizer, text, max_len=100):\n",
    "    if isinstance(text,str):\n",
    "        output=tokenizer.encode_plus(text,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "        input_ids=output[\"input_ids\"].squeeze(0)\n",
    "        token_type_ids=output[\"token_type_ids\"].squeeze(0)\n",
    "        attention_mask=output[\"attention_mask\"].squeeze(0)\n",
    "    elif isinstance(text,list):\n",
    "        input_ids,token_type_ids,attention_mask=[],[],[]\n",
    "        for e in text:\n",
    "            output=tokenizer.encode_plus(e,max_length=max_len,pad_to_max_length=True,return_tensors=\"pt\")\n",
    "            input_ids.append(output[\"input_ids\"].squeeze(0))\n",
    "            token_type_ids.append(output[\"token_type_ids\"].squeeze(0))\n",
    "            attention_mask.append(output[\"attention_mask\"].squeeze(0))\n",
    "    else:\n",
    "        raise Exception('type error')\n",
    "    return torch.stack(input_ids).long(),torch.stack(token_type_ids).long(),torch.stack(attention_mask).long()        \n",
    "class RelDataset(Data.Dataset):\n",
    "    def __init__(self,examples):\n",
    "        self.input_ids=torch.stack([e['input_ids'] for e in examples]).long()\n",
    "        self.token_type_ids=torch.stack([e['token_type_ids'] for e in examples]).long()\n",
    "        self.attention_mask=torch.stack([e['attention_mask'] for e in examples]).long()\n",
    "        self.rel_label=torch.stack([e['rel_label'] for e in examples]).long()\n",
    "        self.postag=torch.stack([e['postag'] for e in examples]).long()\n",
    "        self.feature=torch.stack([e['feature'] for e in examples]).float()\n",
    "        self.token_vec=np.stack([np.array(e['token_vec']) for e in examples])\n",
    "        self.word_vec=np.stack([np.array(e['word_vec']) for e in examples])\n",
    "        self.word_mask=np.stack([np.array(e['word_mask']) for e in examples])\n",
    "        self.plan_label=np.stack([np.array(e['plan_label']) for e in examples])\n",
    "        self.token2docs=[e[\"token2doc\"] for e in examples]\n",
    "    def __len__(self):\n",
    "        return self.input_ids.shape[0]\n",
    "    def __getitem__(self,idx):\n",
    "        return self.input_ids[idx],self.attention_mask[idx],self.token_type_ids[idx],\\\n",
    "               self.rel_label[idx],self.postag[idx],self.feature[idx],self.token_vec[idx],self.word_vec[idx],\\\n",
    "               self.word_mask[idx],self.plan_label[idx],self.token2docs[idx]  \n",
    "class NerDataset(Data.Dataset):\n",
    "    def __init__(self,examples):\n",
    "        self.input_ids=torch.stack([e['input_ids'] for e in examples]).long()\n",
    "        self.token_type_ids=torch.stack([e['token_type_ids'] for e in examples]).long()\n",
    "        self.attention_mask=torch.stack([e['attention_mask'] for e in examples]).long()\n",
    "        self.labels=torch.stack([e['labels'] for e in examples]).long()\n",
    "        self.rel_label=torch.stack([e['rel_label'] for e in examples]).long()\n",
    "        self.token2docs=[e[\"token2doc\"] for e in examples]\n",
    "    def __len__(self):\n",
    "        return self.input_ids.shape[0]\n",
    "    def __getitem__(self,idx):\n",
    "        return self.input_ids[idx],self.attention_mask[idx],self.token_type_ids[idx],self.rel_label[idx],self.labels[idx],self.token2docs[idx]  \n",
    "import unicodedata\n",
    "def _is_whitespace(char):\n",
    "    \"\"\"Checks whether `chars` is a whitespace character.\"\"\"\n",
    "    # \\t, \\n, and \\r are technically contorl characters but we treat them\n",
    "    # as whitespace since they are generally considered as such.\n",
    "    if char == \" \" or char == \"\\t\" or char == \"\\n\" or char == \"\\r\":\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat == \"Zs\":\n",
    "        return True\n",
    "    return False\n",
    "def is_chinese_char(cp):\n",
    "    \"\"\"Checks whether CP is the codepoint of a CJK character.\"\"\"\n",
    "    # This defines a \"chinese character\" as anything in the CJK Unicode block:\n",
    "    #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)\n",
    "    #\n",
    "    # Note that the CJK Unicode block is NOT all Japanese and Korean characters,\n",
    "    # despite its name. The modern Korean Hangul alphabet is a different block,\n",
    "    # as is Japanese Hiragana and Katakana. Those alphabets are used to write\n",
    "    # space-separated words, so they are not treated specially and handled\n",
    "    # like the all of the other languages.\n",
    "    if (\n",
    "        (cp >= 0x4E00 and cp <= 0x9FFF)\n",
    "        or (cp >= 0x3400 and cp <= 0x4DBF)  #\n",
    "        or (cp >= 0x20000 and cp <= 0x2A6DF)  #\n",
    "        or (cp >= 0x2A700 and cp <= 0x2B73F)  #\n",
    "        or (cp >= 0x2B740 and cp <= 0x2B81F)  #\n",
    "        or (cp >= 0x2B820 and cp <= 0x2CEAF)  #\n",
    "        or (cp >= 0xF900 and cp <= 0xFAFF)\n",
    "        or (cp >= 0x2F800 and cp <= 0x2FA1F)  #\n",
    "    ):  #\n",
    "        return True\n",
    "\n",
    "    return False\n",
    "def is_punctuation(char):\n",
    "    \"\"\"Checks whether `chars` is a punctuation character.\"\"\"\n",
    "    cp = ord(char)\n",
    "    # We treat all non-letter/number ASCII as punctuation.\n",
    "    # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n",
    "    # Punctuation class but we treat them as punctuation anyways, for\n",
    "    # consistency.\n",
    "    if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):\n",
    "        return True\n",
    "    cat = unicodedata.category(char)\n",
    "    if cat.startswith(\"P\"):\n",
    "        return True\n",
    "    return False\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "def build_tfidf_svd_matrix(texts,n_output):\n",
    "    corpus=[]\n",
    "    for text in tqdm(texts):\n",
    "#         print(text)\n",
    "        words=word_segment(str(text['text']))\n",
    "#         print(words)\n",
    "        use_words=[]\n",
    "        for word in words:\n",
    "            if word in stop_words:\n",
    "                continue\n",
    "            use_words.append(word)\n",
    "#         print(use_words)\n",
    "        corpus.append(\" \".join(use_words))\n",
    "    tfidf_vec=TfidfVectorizer()\n",
    "    tfidf_matrix=tfidf_vec.fit_transform(corpus)\n",
    "    svd=TruncatedSVD(n_components=n_output,n_iter=7,random_state=42)\n",
    "    tf_idf_svd=svd.fit_transform(tfidf_matrix)\n",
    "    return tf_idf_svd,tfidf_vec,svd\n",
    "def word_segment(sentence):\n",
    "    words=jieba.cut(sentence)\n",
    "    return \",\".join(words).split(\",\")\n",
    "stop_words=set()\n",
    "def load_stopwords():\n",
    "    with open(\"./middle_data/stopwords.txt\",\"r\",encoding=\"UTF-8\") as r:\n",
    "        for line in r.readlines():\n",
    "            stop_words.add(line.strip())\n",
    "load_stopwords()\n",
    "def remove_stopwords(word_list):\n",
    "    res=[]\n",
    "    for word in word_lists:\n",
    "        if word not in stop_words:\n",
    "            res.append(word)\n",
    "    return ' '.join(res)\n",
    "def clean_text(string):\n",
    "    return string.replace(' ', '').replace('\\n', '').replace('\\u3000', '')\n",
    "def index_token(text_raw):\n",
    "    sub_text = []\n",
    "    buff = \"\"\n",
    "    for char in text_raw:\n",
    "        if is_chinese_char(ord(char)) or is_punctuation(char):\n",
    "            if buff != \"\":\n",
    "                sub_text.append(buff)\n",
    "                buff = \"\"\n",
    "            sub_text.append(char)\n",
    "        else:\n",
    "            buff += char\n",
    "    if buff != \"\":\n",
    "        sub_text.append(buff)\n",
    "    tok_to_orig_start_index = []\n",
    "    tok_to_orig_end_index = []\n",
    "    orig_to_tok_index = []\n",
    "    tokens = []\n",
    "    text_tmp = ''\n",
    "    for (i, token) in enumerate(sub_text):\n",
    "        orig_to_tok_index.append(len(tokens))\n",
    "        sub_tokens = tokenizer.tokenize(token)\n",
    "        text_tmp += token\n",
    "        for sub_token in sub_tokens:\n",
    "            tok_to_orig_start_index.append(len(text_tmp) - len(token))\n",
    "            tok_to_orig_end_index.append(len(text_tmp) - 1)\n",
    "            tokens.append(sub_token)\n",
    "        else:\n",
    "            continue\n",
    "        break\n",
    "    return tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/15/2020 19:01:40 - INFO - transformers.tokenization_utils -   Model name './roberta-zh-wwm-pytorch/vocab.txt' not found in model shortcut name list (bert-base-uncased, bert-large-uncased, bert-base-cased, bert-large-cased, bert-base-multilingual-uncased, bert-base-multilingual-cased, bert-base-chinese, bert-base-german-cased, bert-large-uncased-whole-word-masking, bert-large-cased-whole-word-masking, bert-large-uncased-whole-word-masking-finetuned-squad, bert-large-cased-whole-word-masking-finetuned-squad, bert-base-cased-finetuned-mrpc, bert-base-german-dbmdz-cased, bert-base-german-dbmdz-uncased, bert-base-finnish-cased-v1, bert-base-finnish-uncased-v1, bert-base-dutch-cased). Assuming './roberta-zh-wwm-pytorch/vocab.txt' is a path, a model identifier, or url to a directory containing tokenizer files.\n",
      "05/15/2020 19:01:40 - WARNING - transformers.tokenization_utils -   Calling BertTokenizer.from_pretrained() with the path to a single file or url is deprecated\n",
      "05/15/2020 19:01:40 - INFO - transformers.tokenization_utils -   loading file ./roberta-zh-wwm-pytorch/vocab.txt\n"
     ]
    }
   ],
   "source": [
    "do_lower_case=True\n",
    "max_len=256\n",
    "device=torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "# bert_dir=\"./bert-pytorch-chinese/\"\n",
    "# vocab=\"bert-base-chinese-vocab.txt\"\n",
    "# config_file=\"bert_config.json\"\n",
    "bert_dir=\"./roberta-zh-wwm-pytorch/\"\n",
    "vocab=\"vocab.txt\"\n",
    "config_file=\"bert_config.json\"\n",
    "tokenizer=BertTokenizer.from_pretrained(os.path.join(bert_dir,vocab),do_lower_case=do_lower_case)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "code_folding": [
     1,
     6,
     22,
     27,
     33,
     40,
     47,
     53,
     83,
     114
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading dict...\n",
      "loading...\n"
     ]
    }
   ],
   "source": [
    "# text_data=[]\n",
    "# with open(\"./dataset/train_data/train_data.json\",\"r\") as r:\n",
    "#     raw_data=r.readlines()\n",
    "#     for d in raw_data:\n",
    "#         text_data.append(json.loads(d))\n",
    "# print(len(text_data))\n",
    "# with open(\"./dataset/dev_data/dev_data.json\",\"r\") as r:\n",
    "#     raw_data=r.readlines()\n",
    "#     for d in raw_data:\n",
    "#         text_data.append(json.loads(d))\n",
    "# print(len(text_data))\n",
    "text_data=[]\n",
    "with open(\"./dataset/train_data/new_train_data.json\",\"r\") as r:\n",
    "    raw_data=r.readlines()\n",
    "    for d in raw_data:\n",
    "        text_data.append(json.loads(d))\n",
    "# test_text_data=[]\n",
    "# with open(\"./dataset/test1_data/test1_data.json\",\"r\") as r:\n",
    "#     raw_data=r.readlines()\n",
    "#     for d in raw_data:\n",
    "#         test_text_data.append(json.loads(d))\n",
    "test_text_data=[]\n",
    "with open(\"./dataset/test1_data/new_test1_data.json\",\"r\") as r:\n",
    "    raw_data=r.readlines()\n",
    "    for d in raw_data:\n",
    "        test_text_data.append(json.loads(d))\n",
    "schema=[]\n",
    "with open(\"./dataset/schema.json\",\"r\") as r:\n",
    "    raw_schema=r.readlines()\n",
    "    for d in raw_schema:\n",
    "        schema.append(json.loads(d))\n",
    "rels=set()\n",
    "special_rels=set()\n",
    "for e in schema:\n",
    "        if len(e['object_type'].keys())==1:\n",
    "            rels.add(e[\"predicate\"])\n",
    "        else:\n",
    "            special_rels.add(e[\"predicate\"])\n",
    "            for key in e['object_type'].keys():\n",
    "                rels.add(e['predicate']+\"_\"+key)\n",
    "if not os.path.exists(\"./dataset/dict.pk\"):\n",
    "    special_rels=list(special_rels)\n",
    "    id2rels=list(rels)\n",
    "    rels2id=dict([(rel,idx) for idx,rel in enumerate(id2rels)])\n",
    "    id2labels=[\"O\",\"B-OBJ\",\"I-OBJ\",\"B-SUB\",\"I-SUB\",\"[category]\",\"[SEP]\",\"[CLS]\",\"[PAD]\"]\n",
    "    label2ids=dict([ (label,idx) for idx,label in enumerate(id2labels)])\n",
    "    pk.dump([special_rels,id2rels,rels2id,id2labels,label2ids],open(\"./dataset/dict.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading dict...\")\n",
    "    special_rels,id2rels,rels2id,id2labels,label2ids=pk.load(open(\"./dataset/dict.pk\",\"rb\"))\n",
    "id2kglabels=['O','I']+['B-'+e+\"-SUB\" for e in id2rels]+['B-'+e+\"-OB\" for e in id2rels]\n",
    "kglabels2id=dict([ (label,idx) for idx,label in enumerate(id2kglabels)])\n",
    "id2reltype=[[] for i in range(len(id2rels))]\n",
    "for e in schema:\n",
    "    if len(e['object_type'].keys())==1:\n",
    "        rel=e[\"predicate\"]\n",
    "        ids=rels2id[rel]\n",
    "        id2reltype[ids].append(e)\n",
    "    else:\n",
    "        for key in e['object_type'].keys():\n",
    "            rel=e['predicate']+\"_\"+key\n",
    "            ids=rels2id[rel]\n",
    "            temp_e=copy.deepcopy(e)\n",
    "            poped_keys=[]\n",
    "            for k in temp_e['object_type'].keys():\n",
    "                if k!=key:\n",
    "                    poped_keys.append(k)\n",
    "            for k in poped_keys:\n",
    "                 temp_e['object_type'].pop(k)\n",
    "            id2reltype[ids].append(temp_e)\n",
    "id2schema=[e[0] for e in id2reltype]\n",
    "id2rel_text=[[] for i in range(len(id2rels))]\n",
    "id2rel_rawtext=[[] for i in range(len(id2rels))]\n",
    "id2rel_token2text=[[] for i in range(len(id2rels))]\n",
    "for rel in range(len(id2rels)):\n",
    "    if id2rels[rel].split(\"_\")[0] not in special_rels:\n",
    "        cls_text=id2schema[rel]['subject_type']+\",\"+id2schema[rel]['predicate']+\",\"+id2schema[rel]['object_type']['@value']\n",
    "    else:\n",
    "        cls_text=id2schema[rel]['subject_type']+\",\"+id2schema[rel]['predicate']+\",\"+id2schema[rel]['object_type'][id2rels[rel].split(\"_\")[1]]\n",
    "    id2rel_text[rel]=tokenizer.tokenize(cls_text)\n",
    "    id2rel_rawtext[rel]=cls_text\n",
    "    id2rel_token2text[rel]=index_token(cls_text)[0]\n",
    "    assert len(id2rel_token2text[rel])==len(id2rel_text[rel])\n",
    "if not os.path.exists(\"./middle_data/rel_data_postag.pk\"):\n",
    "    jieba.enable_paddle() \n",
    "    jieba.enable_parallel(8)\n",
    "    rel_cut_words=[]\n",
    "    rel_cut_tags=[]\n",
    "    for idx in tqdm(range(len(id2rel_rawtext))):\n",
    "        words = pseg.lcut(id2rel_rawtext[idx],use_paddle=True) #jieba默认模式\n",
    "        new_words=[w for w,t in words]\n",
    "        new_tags=[t for w,t in words]\n",
    "        rel_cut_words.append([idx,new_words])\n",
    "        rel_cut_tags.append([idx,new_tags])\n",
    "    rel_cut_words=[e[1] for e in sorted(rel_cut_words,key=lambda x:x[0])]\n",
    "    rel_cut_tags=[e[1] for e in sorted(rel_cut_tags,key=lambda x:x[0])]\n",
    "    rel_data_postag=[]\n",
    "    for idx in tqdm(range(len(id2rel_rawtext))):\n",
    "        assert len(id2rel_rawtext[idx].strip())==len(\"\".join(rel_cut_words[idx]))\n",
    "        indexs=[]\n",
    "        cur_length=0\n",
    "        for e in rel_cut_words[idx]:\n",
    "            indexs.append([cur_length,cur_length+len(e)-1])\n",
    "            cur_length+=len(e)\n",
    "        pos_label=np.zeros(len(id2rel_rawtext[idx])).astype(np.int8)\n",
    "        for i,(b,e) in enumerate(indexs):\n",
    "            assert (id2rel_rawtext[idx][b]==rel_cut_words[idx][i][0] or _is_whitespace(id2rel_rawtext[idx].strip()[b])\\\n",
    "                   or _is_whitespace(rel_cut_words[idx][i][0])) \\\n",
    "                    and (id2rel_rawtext[idx].strip()[e]==rel_cut_words[idx][i][-1] \\\n",
    "                         or _is_whitespace(id2rel_rawtext[idx].strip()[e])  or _is_whitespace(rel_cut_words[idx][i][-1]))\n",
    "            pos_label[b+1:e+1]=pos2id_BIO['I-'+rel_cut_tags[idx][i]]\n",
    "            pos_label[b]=pos2id_BIO['B-'+rel_cut_tags[idx][i]]\n",
    "        rel_data_postag.append(pos_label)\n",
    "    pk.dump(rel_data_postag,open(\"./middle_data/rel_data_postag.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    rel_data_postag=pk.load(open(\"./middle_data/rel_data_postag.pk\",\"rb\"))\n",
    "special_major_idx=[2,4,22,32,54]\n",
    "special_affilate_idx=[[] for i in range(len(id2rels))]\n",
    "special_affilate_idx[2]=[5]\n",
    "special_affilate_idx[4]=[0]\n",
    "special_affilate_idx[22]=[51]\n",
    "special_affilate_idx[32]=[8]\n",
    "special_affilate_idx[54]=[6,11,14]\n",
    "special_pass_idx=[0,5,6,8,11,14,51]\n",
    "# [(id2rels[e],[id2rels[t] for t in special_affilate_idx[e]]) for e in special_major_idx],[id2rels[e] for e in special_pass_idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(\"./middle_data/texts_tfidf_svd.pk\"):\n",
    "    texts_tfidf_svd,tfidf_vectorizer,svd_vectorizer=build_tfidf_svd_matrix(text_data,16)\n",
    "    joblib.dump(tfidf_vectorizer,\"./middle_data/tfidf_vectorizer.sk.model\")\n",
    "    joblib.dump(svd_vectorizer,\"./middle_data/svd_vectorizer.sk.model\")\n",
    "    pk.dump(texts_tfidf_svd,open(\"./middle_data/texts_tfidf_svd.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    texts_tfidf_svd=pk.load(open(\"./middle_data/texts_tfidf_svd.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": [
     0,
     26
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n",
      "loading...\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(\"./middle_data/train_postag.pk\"):\n",
    "    postag_set=set()\n",
    "    jieba.enable_parallel(8)\n",
    "    train_cut_words=[]\n",
    "    train_cut_tags=[]\n",
    "    for idx in tqdm(range(len(text_data))):\n",
    "        words = pseg.lcut(text_data[idx]['text'],use_paddle=True) #jieba默认模式\n",
    "        new_words=[w for w,t in words]\n",
    "        new_tags=[t for w,t in words]\n",
    "        postag_set.update(new_tags)\n",
    "        train_cut_words.append([idx,new_words])\n",
    "        train_cut_tags.append([idx,new_tags])\n",
    "    train_cut_words=[e[1] for e in sorted(train_cut_words,key=lambda x:x[0])]\n",
    "    train_cut_tags=[e[1] for e in sorted(train_cut_tags,key=lambda x:x[0])]\n",
    "    id2pos=list(postag_set)\n",
    "    pos2id=dict([(pos,idx) for idx,pos in enumerate(id2pos)])\n",
    "    jieba.enable_paddle() \n",
    "    jieba.enable_parallel(16)\n",
    "    for idx in tqdm(range(len(text_data))):\n",
    "        words = pseg.lcut(text_data[idx]['text'],use_paddle=True) #jieba默认模式\n",
    "        new_words=[w for w,t in words]\n",
    "        assert new_words==train_cut_words[idx]\n",
    "    pk.dump([id2pos,pos2id,train_cut_words,train_cut_tags],open(\"./middle_data/train_postag.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    id2pos,pos2id,train_cut_words,train_cut_tags=pk.load(open(\"./middle_data/train_postag.pk\",\"rb\"))\n",
    "if not os.path.exists(\"./middle_data/train_postag_BIO.pk\"):\n",
    "    id2pos_BIO=['B-'+e for e in id2pos]+['I-'+e for e in id2pos]\n",
    "    id2pos_BIO.extend(['[CLS]','[SEP]','[PAD]'])\n",
    "    pos2id_BIO=dict([(pos,idx) for idx,pos in enumerate(id2pos_BIO)])\n",
    "\n",
    "    text_data_postag=[]\n",
    "    for idx in tqdm(range(len(text_data))):\n",
    "        assert len(text_data[idx]['text'].strip())==len(\"\".join(train_cut_words[idx]))\n",
    "        indexs=[]\n",
    "        cur_length=0\n",
    "        for e in train_cut_words[idx]:\n",
    "            indexs.append([cur_length,cur_length+len(e)-1])\n",
    "            cur_length+=len(e)\n",
    "        pos_label=np.zeros(len(text_data[idx]['text'])).astype(np.int8)\n",
    "        for i,(b,e) in enumerate(indexs):\n",
    "            assert (text_data[idx]['text'].strip()[b]==train_cut_words[idx][i][0] or _is_whitespace(text_data[idx]['text'].strip()[b])\\\n",
    "                   or _is_whitespace(train_cut_words[idx][i][0])) \\\n",
    "                    and (text_data[idx]['text'].strip()[e]==train_cut_words[idx][i][-1] \\\n",
    "                         or _is_whitespace(text_data[idx]['text'].strip()[e])  or _is_whitespace(train_cut_words[idx][i][-1]))\n",
    "            pos_label[b+1:e+1]=pos2id_BIO['I-'+train_cut_tags[idx][i]]\n",
    "            pos_label[b]=pos2id_BIO['B-'+train_cut_tags[idx][i]]\n",
    "        text_data_postag.append(pos_label)\n",
    "    pk.dump([id2pos_BIO,pos2id_BIO,text_data_postag],open(\"./middle_data/train_postag_BIO.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    id2pos_BIO,pos2id_BIO,text_data_postag=pk.load(open(\"./middle_data/train_postag_BIO.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/15/2020 19:02:17 - INFO - gensim.utils -   loading Word2VecKeyedVectors object from ./Tencent_ChineseEmbedding/ChineseEmbedding.bin\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/15/2020 19:02:46 - INFO - gensim.utils -   loading vectors from ./Tencent_ChineseEmbedding/ChineseEmbedding.bin.vectors.npy with mmap=r\n",
      "05/15/2020 19:02:46 - INFO - gensim.utils -   setting ignored attribute vectors_norm to None\n",
      "05/15/2020 19:02:46 - INFO - gensim.utils -   loaded ./Tencent_ChineseEmbedding/ChineseEmbedding.bin\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists('./Tencent_ChineseEmbedding/ChineseEmbedding.bin'):\n",
    "    file = './Tencent_ChineseEmbedding/Tencent_AILab_ChineseEmbedding.txt'\n",
    "    wv_from_text = KeyedVectors.load_word2vec_format(file, binary=False)\n",
    "    wv_from_text.init_sims(replace=True)\n",
    "    # 重新保存加载变量为二进制形式\n",
    "    wv_from_text.save('./Tencent_ChineseEmbedding/ChineseEmbedding.bin')\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    wv_from_text = gensim.models.KeyedVectors.load('./Tencent_ChineseEmbedding/ChineseEmbedding.bin', mmap='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": [
     0,
     9
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading extra tokens vector file...\n",
      "loading...\n"
     ]
    }
   ],
   "source": [
    "if not os.path.exists(\"./Tencent_ChineseEmbedding/extra_embedding.npy\"):\n",
    "    word2vec_dims=wv_from_text.vectors.shape[-1]\n",
    "    pad_vec=np.zeros([1,word2vec_dims],dtype=np.float32)\n",
    "    other_token_vec=np.random.normal(scale=0.02,size=[3,word2vec_dims])\n",
    "    extra_token_vec=np.concatenate([pad_vec,other_token_vec])\n",
    "    np.save(\"./Tencent_ChineseEmbedding/extra_embedding.npy\",extra_token_vec)\n",
    "else:\n",
    "    print(\"loading extra tokens vector file...\")\n",
    "    extra_token_vec=np.load(\"./Tencent_ChineseEmbedding/extra_embedding.npy\")\n",
    "if not os.path.exists(\"./Tencent_ChineseEmbedding/word_id.pk\"):\n",
    "    word2id={}\n",
    "    id2word=[[] for i in range(word2vec.shape[0])]\n",
    "    for i,key in tqdm(enumerate(keys)):\n",
    "        assert (word2vec[4+i]==wv_from_text[key]).all()\n",
    "        word2id[key]=4+i\n",
    "        id2word[4+i]=key\n",
    "    id2word[0]=tokenizer.pad_token\n",
    "    word2id[tokenizer.pad_token]=0\n",
    "    id2word[1]=tokenizer.unk_token\n",
    "    word2id[tokenizer.unk_token]=1\n",
    "    id2word[2]=tokenizer.cls_token\n",
    "    word2id[tokenizer.cls_token]=2\n",
    "    id2word[3]=tokenizer.sep_token\n",
    "    word2id[tokenizer.sep_token]=3\n",
    "    pk.dump([word2id,id2word],open(\"./Tencent_ChineseEmbedding/word_id.pk\",\"wb\"),protocol = 4)\n",
    "else:\n",
    "    print(\"loading...\")\n",
    "    word2id,id2word=pk.load(open(\"./Tencent_ChineseEmbedding/word_id.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2vec=np.concatenate([extra_token_vec,wv_from_text.vectors])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": [
     0,
     18
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n",
      "loading...\n"
     ]
    }
   ],
   "source": [
    "# if not os.path.exists(\"./middle_data/train_wordvec.pk\"):\n",
    "#     jieba.enable_parallel(8)\n",
    "#     train_cut_words=[]\n",
    "#     train_cut_vecs=[]\n",
    "#     for idx in tqdm(range(len(text_data))):\n",
    "#         word_list=jieba.lcut(text_data[idx]['text'],HMM=False,use_paddle=False)\n",
    "#         wvid_list=[ word2id.get(e,1)  for e in word_list ]\n",
    "#         train_cut_words.append([idx,word_list])\n",
    "#         train_cut_vecs.append([idx,wvid_list])\n",
    "#     train_cut_words=[e[1] for e in sorted(train_cut_words,key=lambda x:x[0])]\n",
    "#     train_cut_vecs=[e[1] for e in sorted(train_cut_vecs,key=lambda x:x[0])]\n",
    "# #     train_cut_wvs=[]\n",
    "# #     for e in tqdm(train_cut_vecs):\n",
    "# #         train_cut_wvs.append(np.array([word2vec[t] for t in e]))\n",
    "#     pk.dump([train_cut_words,train_cut_vecs],open(\"./middle_data/train_wordvec.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading...\")\n",
    "#     train_cut_words,train_cut_vecs=pk.load(open(\"./middle_data/train_wordvec.pk\",\"rb\"))\n",
    "# if not os.path.exists(\"./middle_data/train_tokenvec.pk\"):\n",
    "#     text_data_tokenvec=[]\n",
    "#     for idx in tqdm(range(len(text_data))):\n",
    "#         assert len(text_data[idx]['text'])==len(\"\".join(train_cut_words[idx]))\n",
    "#         indexs=[]\n",
    "#         cur_length=0\n",
    "#         for e in train_cut_words[idx]:\n",
    "#             indexs.append([cur_length,cur_length+len(e)-1])\n",
    "#             cur_length+=len(e)\n",
    "#         vec_label=np.zeros((len(text_data[idx]['text']))).astype(np.int)\n",
    "#         for i,(b,e) in enumerate(indexs):\n",
    "#             assert (text_data[idx]['text'][b]==train_cut_words[idx][i][0] or _is_whitespace(text_data[idx]['text'][b])\\\n",
    "#                    or _is_whitespace(train_cut_words[idx][i][0])) \\\n",
    "#                     and (text_data[idx]['text'][e]==train_cut_words[idx][i][-1] \\\n",
    "#                          or _is_whitespace(text_data[idx]['text'][e])  or _is_whitespace(train_cut_words[idx][i][-1]))\n",
    "#             vec_label[b:e+1]=train_cut_vecs[idx][i]\n",
    "#         text_data_tokenvec.append(vec_label)\n",
    "#     pk.dump(text_data_tokenvec,open(\"./middle_data/train_tokenvec.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading...\")\n",
    "#     text_data_tokenvec=pk.load(open(\"./middle_data/train_tokenvec.pk\",\"rb\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n"
     ]
    }
   ],
   "source": [
    "# if not os.path.exists(\"./middle_data/doc_cls_info.pk\"):\n",
    "#     cls_doc_tokens=[]\n",
    "#     cls_doc_rels=[]\n",
    "#     cls_token2doc=[]\n",
    "#     for e in tqdm(text_data):\n",
    "#         tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index,tokens,rel_labels=_convert_example_to_cls_record(e,tokenizer)\n",
    "#         cls_doc_tokens.append(tokens)\n",
    "#         cls_doc_rels.append(rel_labels)\n",
    "#         cls_token2doc.append([tok_to_orig_start_index,tok_to_orig_end_index,orig_to_tok_index])\n",
    "#     pk.dump([cls_doc_rels,cls_doc_tokens,cls_token2doc],open(\"./middle_data/doc_cls_info.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading...\")\n",
    "#     cls_doc_rels,cls_doc_tokens,cls_token2doc=pk.load(open(\"./middle_data/doc_cls_info.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading...\n"
     ]
    }
   ],
   "source": [
    "# if not os.path.exists(\"./middle_data/train_plan_labels.pk\"):\n",
    "#     train_plan_labels=[]\n",
    "#     for e in tqdm(text_data):\n",
    "#         train_plan_labels.append(_convert_example_to_plan_labels(e,tokenizer))\n",
    "#     for idx in tqdm(range(len(train_plan_labels))):\n",
    "#         for jdx in range(len(train_plan_labels[idx])):\n",
    "#             if sum(train_plan_labels[idx][jdx])==0:\n",
    "#                 train_plan_labels[idx][jdx][0]=1\n",
    "#     pk.dump(train_plan_labels,open(\"./middle_data/train_plan_labels.pk\",\"wb\"))\n",
    "# else:\n",
    "#     print(\"loading...\")\n",
    "#     train_plan_labels=pk.load(open(\"./middle_data/train_plan_labels.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "code_folding": [
     0,
     47
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading example..\n"
     ]
    }
   ],
   "source": [
    "def create_cls_example(tokens,rel,token2doc,postag,tfidf_svd,word_vec,token_vec,plan_label,tokenizer):\n",
    "        word_vec=copy.deepcopy(word_vec)\n",
    "        token_vec=copy.deepcopy(token_vec)\n",
    "        if len(tokens)>max_len-2:\n",
    "                tokens=tokens[:(max_len-2)]\n",
    "                plan_label=plan_label[:(max_len-2)]\n",
    "                token2doc=[e[:(max_len-2)] if ind<2 else e for ind,e in enumerate(token2doc)]\n",
    "        tag=[postag[token2doc[0][idx]] for idx in range(len(tokens))]\n",
    "        tokenvec=[token_vec[token2doc[0][idx]] for idx in range(len(tokens))]\n",
    "        tag=[pos2id_BIO[tokenizer.cls_token]]+tag+[pos2id_BIO[tokenizer.sep_token]]\n",
    "        tokenvec=[word2id[tokenizer.cls_token]]+tokenvec+[word2id[tokenizer.sep_token]]\n",
    "        full_tokens=[tokenizer.cls_token]+tokens+[tokenizer.sep_token]\n",
    "        padding_O=[0]*len(id2kglabels)\n",
    "        padding_O[0]=1\n",
    "        full_plan_labels=[padding_O]+plan_label+[padding_O]\n",
    "        token_type_ids=[0]*len(full_tokens)\n",
    "        attention_mask=[1]*len(token_type_ids)\n",
    "        cur_len=len(full_tokens)\n",
    "        if cur_len<max_len:\n",
    "            full_tokens+=[tokenizer.pad_token]*(max_len-cur_len)\n",
    "            full_plan_labels+=[padding_O]*(max_len-cur_len)\n",
    "            attention_mask+=[0]*(max_len-cur_len)\n",
    "            token_type_ids+=[0]*(max_len-cur_len)\n",
    "            tag+=[pos2id_BIO[tokenizer.pad_token]]*(max_len-cur_len)\n",
    "            tokenvec+=[word2id[tokenizer.pad_token]]*(max_len-cur_len)\n",
    "\n",
    "        if len(word_vec)>max_len//2:\n",
    "            word_vec=word_vec[:max_len//2]\n",
    "        word_mask=[1]*len(word_vec)\n",
    "        if len(word_vec)<max_len//2:\n",
    "            word_mask+=[0]*(max_len//2-len(word_vec))\n",
    "            word_vec+=[word2id[tokenizer.pad_token]]*(max_len//2-len(word_vec))\n",
    "\n",
    "        full_ids=tokenizer.convert_tokens_to_ids(full_tokens)\n",
    "        if len(rel)>0:\n",
    "            example={\"input_ids\":torch.tensor(full_ids,dtype=torch.long),\"token_type_ids\":torch.tensor(token_type_ids,dtype=torch.long),\\\n",
    "                    \"attention_mask\":torch.tensor(attention_mask,dtype=torch.long),\n",
    "                    \"rel_label\":(F.one_hot(torch.tensor(rel),len(id2rels)).sum(dim=0)!=0).long(),\n",
    "                     \"postag\":torch.tensor(tag).long(),\"feature\":torch.tensor(tfidf_svd).float(),\\\n",
    "                     \"token_vec\":tokenvec,\"word_vec\":word_vec,\"word_mask\":word_mask,\"plan_label\":full_plan_labels,\"token2doc\":token2doc}\n",
    "        else:\n",
    "            example={\"input_ids\":torch.tensor(full_ids,dtype=torch.long),\"token_type_ids\":torch.tensor(token_type_ids,dtype=torch.long),\\\n",
    "                    \"attention_mask\":torch.tensor(attention_mask,dtype=torch.long),\n",
    "                     \"rel_label\":torch.zeros(len(id2rels)).long(),\n",
    "                     \"postag\":torch.tensor(tag).long(),\"feature\":torch.tensor(tfidf_svd).float(),\\\n",
    "                     \"token_vec\":tokenvec,\"word_vec\":word_vec,\"word_mask\":word_mask,\"plan_label\":full_plan_labels,\"token2doc\":token2doc}  \n",
    "        return example\n",
    "if not os.path.exists(\"./middle_data/cls_examples.pk\"):\n",
    "    cls_examples,train_wordvecs,train_tokenvecs=[],[],[]\n",
    "    for idx,(tokens,rel,token2doc,postag) in tqdm(enumerate(zip(cls_doc_tokens,cls_doc_rels,cls_token2doc,text_data_postag))):\n",
    "                example=create_cls_example(tokens,rel,token2doc,postag,texts_tfidf_svd[idx],\\\n",
    "                                                       train_cut_vecs[idx],text_data_tokenvec[idx],train_plan_labels[idx],tokenizer)\n",
    "                cls_examples.append(example)\n",
    "    pk.dump(cls_examples,open(\"./middle_data/cls_examples.pk\",\"wb\"))\n",
    "else:\n",
    "    print(\"loading example..\")\n",
    "    cls_examples=pk.load(open(\"./middle_data/cls_examples.pk\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "191967"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(cls_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "spliter=KFold(n_splits=4,shuffle=True,random_state=20)\n",
    "for x,y in spliter.split(cls_examples):\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_examples=np.array(cls_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((143975,), (47992,))"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cls_examples[x].shape,cls_examples[y].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(256, 112)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(cls_examples[0]['plan_label']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "cls_train_dataset=RelDataset(cls_examples[:10000])\n",
    "cls_val_dataset=RelDataset(cls_examples[-1000:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "hidden_dropout_prob = 0.1\n",
    "num_rel_labels = len(id2rels)\n",
    "num_token_labels=len(id2labels)\n",
    "learning_rate = 5e-5\n",
    "weight_decay = 0\n",
    "epochs =3\n",
    "batch_size = 16\n",
    "adam_epsilon=1e-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir=\"./output_test/\"\n",
    "args=ARG(train_batch_size=batch_size*2,eval_batch_size=batch_size*2,weight_decay=weight_decay,learning_rate=learning_rate,\n",
    "         adam_epsilon=adam_epsilon,num_train_epochs=epochs,warmup_steps=0,gradient_accumulation_steps=1,save_steps=313,\n",
    "         max_grad_norm=1.0,model_name_or_path=output_dir,output_dir=output_dir,seed=42,device=device,n_gpu=2,\n",
    "        max_steps=0,output_mode=\"classification\",fp16=False,fp16_opt_level='O1',card_list=[0,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "code_folding": [
     0,
     14,
     33,
     54,
     167,
     215,
     221,
     391
    ]
   },
   "outputs": [],
   "source": [
    "class clsLoss(nn.Module):\n",
    "    def __init__(self,m,alpha):\n",
    "        super(clsLoss,self).__init__()\n",
    "        self.m=m\n",
    "        self.alpha=alpha\n",
    "    def forward(self,logits,targets):\n",
    "        y_hat=torch.sigmoid(logits)\n",
    "        y=targets\n",
    "        theta=(((y_hat-0.5)*(y.float()-0.5))<0) | ((y_hat>0.5-self.m) & (y_hat<0.5+self.m))\n",
    "        loss_p=-(theta.float()*y.float()*torch.log(y_hat+1e-5)).sum(dim=-1)/y.float().sum(dim=-1)\n",
    "        loss_n=-(theta.float()*(1-y).float()*torch.log((1-y_hat)+1e-5)).sum(dim=-1)/(1-y.float()).sum(dim=-1)\n",
    "#         loss_n=-torch.log(1-(theta.float()*(1-y).float()*y_hat).sum(dim=-1)/(1-y).float().sum(dim=-1))\n",
    "        loss=loss_p+self.alpha*loss_n\n",
    "        return loss.mean()\n",
    "class GCNN_block(nn.Module):\n",
    "    def __init__(self,input_channel,output_channel,kernel_size,padding,dilation=1):\n",
    "        super(GCNN_block,self).__init__()\n",
    "        self.input_channel=input_channel\n",
    "        self.output_channel=output_channel\n",
    "        self.conv1=nn.Conv1d(input_channel,output_channel,kernel_size,padding=padding,dilation=dilation)\n",
    "        self.conv2=nn.Conv1d(input_channel,output_channel,kernel_size,padding=padding,dilation=dilation)\n",
    "        if input_channel !=output_channel:\n",
    "            self.trans=nn.Conv1d(input_channel,output_channel,1)\n",
    "    def forward(self,args):\n",
    "        X,attention_mask=args[0],args[1]\n",
    "        X=X*attention_mask.unsqueeze(1).float()\n",
    "        gate=torch.sigmoid(self.conv2(X))\n",
    "        if self.input_channel==self.output_channel:\n",
    "            Y=X*(1-gate)+self.conv1(X)*gate\n",
    "        else:\n",
    "            Y=self.trans(X)*(1-gate)+self.conv1(X)*gate\n",
    "        Y=Y*attention_mask.unsqueeze(1).float()\n",
    "        return Y,attention_mask\n",
    "class Conditional_LayerNorm(nn.Module):\n",
    "    def __init__(self,features,conditional_dim,eps=1e-6):\n",
    "        super(Conditional_LayerNorm,self).__init__()\n",
    "        self.gamma=nn.Parameter(torch.ones(features))\n",
    "        self.beta=nn.Parameter(torch.zeros(features))\n",
    "        self.trans_gamma=nn.Linear(conditional_dim,self.gamma.shape[-1])\n",
    "        self.trans_beta=nn.Linear(conditional_dim,self.beta.shape[-1])\n",
    "        torch.nn.init.constant_(self.trans_gamma.weight,val=0)\n",
    "        torch.nn.init.constant_(self.trans_gamma.bias,val=0)\n",
    "        torch.nn.init.constant_(self.trans_beta.weight,val=0)\n",
    "        torch.nn.init.constant_(self.trans_beta.bias,val=0)\n",
    "        self.eps=eps\n",
    "    def forward(self,X,condition):\n",
    "        mean=X.mean(-1,keepdim=True)\n",
    "        std=X.std(-1,keepdim=True)\n",
    "        cond_gamma=self.trans_gamma(condition)\n",
    "        cond_beta=self.trans_beta(condition)\n",
    "        if condition.dim()<X.dim(): #condition是固定维度\n",
    "            return (self.gamma+cond_gamma).unsqueeze(1)*(X-mean)/(std+self.eps)+(self.beta+cond_beta).unsqueeze(1)\n",
    "        else:#condition是sequence\n",
    "            return (self.gamma+cond_gamma)*(X-mean)/(std+self.eps)+(self.beta+cond_beta)\n",
    "class BertMulticlass_DGCNN(nn.Module):\n",
    "    def __init__(self,bert_dir,config_file,num_rel_labels,hidden_dropout_prob,embed_dim,\\\n",
    "                 loss_type=True,use_feature=True,use_vec=False,use_word=False,use_pool=False,use_plan=False,alpha=1):\n",
    "        super(BertMulticlass_DGCNN,self).__init__()\n",
    "        self.num_rel_labels = num_rel_labels\n",
    "        self.alpha=alpha\n",
    "        self.bert =BertModel.from_pretrained(bert_dir,config=os.path.join(bert_dir,config_file), \\\n",
    "                                             hidden_dropout_prob=hidden_dropout_prob,output_hidden_states=True, output_attentions=True)\n",
    "        self.dropout = nn.Dropout(hidden_dropout_prob)\n",
    "        self.embed=nn.Embedding(len(id2pos_BIO),embed_dim)\n",
    "        if use_feature:\n",
    "            hidden_size=self.bert.pooler.dense.out_features+embed_dim\n",
    "        else:\n",
    "            hidden_size=self.bert.pooler.dense.out_features      \n",
    "        if use_vec and use_feature:\n",
    "#             hidden_size+=200\n",
    "            self.trans=nn.Linear(200,hidden_size-embed_dim)\n",
    "        if use_vec and not use_feature:\n",
    "            self.trans=nn.Linear(200,hidden_size)\n",
    "        self.DGCNN=nn.Sequential(GCNN_block(hidden_size,hidden_size,1,padding=0,dilation=1),\n",
    "                   GCNN_block(hidden_size,hidden_size,3,padding=1,dilation=1),\n",
    "                   GCNN_block(hidden_size,hidden_size,3,padding=2,dilation=2),\n",
    "                   GCNN_block(hidden_size,hidden_size,3,padding=4,dilation=4))\n",
    "#         self.dense_feature=nn.Sequential(nn.Linear(16,16),nn.ReLU())\n",
    "        logit_size=hidden_size\n",
    "        if use_word:\n",
    "            self.DGCNN_vec=nn.Sequential(GCNN_block(200,200,1,padding=0,dilation=1),\n",
    "                       GCNN_block(200,200,3,padding=1,dilation=1),\n",
    "                       GCNN_block(200,200,3,padding=2,dilation=2),\n",
    "                       GCNN_block(200,200,3,padding=4,dilation=4))\n",
    "            logit_size+=200\n",
    "        if use_plan:\n",
    "            self.cond_ln=Conditional_LayerNorm(hidden_size,112)\n",
    "#             self.DGCNN_plan=nn.Sequential(GCNN_block(112,112,1,padding=0,dilation=1),\n",
    "#                        GCNN_block(112,112,3,padding=1,dilation=1),\n",
    "#                        GCNN_block(112,112,3,padding=2,dilation=2),\n",
    "#                        GCNN_block(112,112,3,padding=4,dilation=4))\n",
    "#             logit_size+=112\n",
    "        if use_pool:#cat or +\n",
    "            logit_size+=self.bert.pooler.dense.out_features \n",
    "        self.classifier=nn.Linear(logit_size,num_rel_labels)\n",
    "        self.loss_type=loss_type\n",
    "        self.use_feature=use_feature\n",
    "        self.use_vec=use_vec\n",
    "        self.use_word=use_word\n",
    "        self.use_pool=use_pool\n",
    "        self.use_plan=use_plan\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        rel_label=None,\n",
    "        feature=None,\n",
    "        postag=None,\n",
    "        tokenvec=None,\n",
    "        wordvec=None,\n",
    "        wordmask=None,\n",
    "        plan_label=None\n",
    "    ):\n",
    "\n",
    "        outputs = self.bert(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids,\n",
    "        )\n",
    "\n",
    "        sequence_output = outputs[0]\n",
    "        pool_output=outputs[1]\n",
    "        if self.use_vec:\n",
    "            tokenvec=self.trans(tokenvec)\n",
    "            sequence_output+=tokenvec\n",
    "#             sequence_output=torch.cat([sequence_output,tokenvec],dim=-1)\n",
    "        if self.use_feature:\n",
    "            postag_output=self.embed(postag)\n",
    "            sequence_output=torch.cat([sequence_output,postag_output],dim=-1)\n",
    "        if self.use_plan:\n",
    "#             plan_output=F.adaptive_avg_pool1d(self.DGCNN_plan([plan_label.permute(0,2,1),attention_mask])[0],1).squeeze(-1)\n",
    "            sequence_output=self.cond_ln(sequence_output,plan_label)\n",
    "        sequence_output=F.adaptive_avg_pool1d(self.DGCNN([sequence_output.permute(0,2,1),attention_mask])[0],1).squeeze(-1)\n",
    "        sequence_output = self.dropout(sequence_output)\n",
    "        if self.use_word:\n",
    "            word_output=F.adaptive_avg_pool1d(self.DGCNN_vec([wordvec.permute(0,2,1),wordmask])[0],1).squeeze(-1)\n",
    "            word_output=self.dropout(word_output)\n",
    "#         if self.use_plan:\n",
    "#             plan_output=F.adaptive_avg_pool1d(self.DGCNN_plan([plan_label.permute(0,2,1),attention_mask])[0],1).squeeze(-1)\n",
    "#             plan_output=self.dropout(plan_output)\n",
    "        logits_input=sequence_output\n",
    "        if self.use_word:\n",
    "            logits_input=torch.cat([logits_input,word_output],dim=-1)\n",
    "#         if self.use_plan:\n",
    "#             logits_input=torch.cat([logits_input,plan_output],dim=-1)    \n",
    "        if self.use_pool:\n",
    "            pool_output=self.dropout(pool_output)\n",
    "            logits_input=torch.cat([logits_input,pool_output],dim=-1) \n",
    "        logits = self.classifier(logits_input)\n",
    "        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here\n",
    "\n",
    "        if rel_label is not None:\n",
    "            if self.num_rel_labels == 1:\n",
    "                #  We are doing regression\n",
    "                loss_fct = MSELoss()\n",
    "                loss = loss_fct(logits.view(-1), labels.view(-1))\n",
    "            else:\n",
    "                if self.loss_type:\n",
    "                    loss_fct = clsLoss(0.3,self.alpha)\n",
    "                    loss = loss_fct(logits.view(-1, self.num_rel_labels), rel_label.view(-1,self.num_rel_labels))\n",
    "                else:\n",
    "                    loss_fct = nn.BCEWithLogitsLoss(reduction=\"none\")\n",
    "                    loss = ((loss_fct(logits.view(-1, self.num_rel_labels), rel_label.view(-1,self.num_rel_labels).float())).sum(dim=-1)).mean()\n",
    "            outputs = (loss,) + outputs\n",
    "\n",
    "        return outputs  # (loss), logits, (hidden_states), (attentions)\n",
    "class BertMulticlass(BertPreTrainedModel):\n",
    "    def __init__(self, config):\n",
    "        super().__init__(config)\n",
    "        self.num_labels = config.num_labels\n",
    "\n",
    "        self.bert = BertModel(config)\n",
    "        self.dropout = nn.Dropout(config.hidden_dropout_prob)\n",
    "        self.classifier = nn.Linear(config.hidden_size, self.config.num_labels)\n",
    "\n",
    "        self.init_weights()\n",
    "    def forward(\n",
    "        self,\n",
    "        input_ids=None,\n",
    "        attention_mask=None,\n",
    "        token_type_ids=None,\n",
    "        position_ids=None,\n",
    "        head_mask=None,\n",
    "        inputs_embeds=None,\n",
    "        labels=None,\n",
    "    ):\n",
    "\n",
    "        outputs = self.bert(\n",
    "            input_ids,\n",
    "            attention_mask=attention_mask,\n",
    "            token_type_ids=token_type_ids,\n",
    "            position_ids=position_ids,\n",
    "            head_mask=head_mask,\n",
    "            inputs_embeds=inputs_embeds,\n",
    "        )\n",
    "\n",
    "        pooled_output = outputs[1]\n",
    "\n",
    "        pooled_output = self.dropout(pooled_output)\n",
    "        logits = self.classifier(pooled_output)\n",
    "\n",
    "        outputs = (logits,) + outputs[2:]  # add hidden states and attention if they are here\n",
    "\n",
    "        if labels is not None:\n",
    "            if self.num_labels == 1:\n",
    "                #  We are doing regression\n",
    "                loss_fct = MSELoss()\n",
    "                loss = loss_fct(logits.view(-1), labels.view(-1))\n",
    "            else:\n",
    "                loss_fct = nn.BCEWithLogitsLoss(reduction=\"none\")\n",
    "                loss = ((loss_fct(logits.view(-1, self.num_labels), labels.view(-1,self.num_labels).float())).sum(dim=-1)).mean()\n",
    "            outputs = (loss,) + outputs\n",
    "\n",
    "        return outputs  # (loss), logits, (hidden_states), (attentions)\n",
    "def set_seed(args):\n",
    "    random.seed(args.seed)\n",
    "    np.random.seed(args.seed)\n",
    "    torch.manual_seed(args.seed)\n",
    "    if args.n_gpu > 0:\n",
    "        torch.cuda.manual_seed_all(args.seed)\n",
    "def train(args, train_dataset,val_dataset, model, tokenizer):\n",
    "#     \"\"\" Train the model \"\"\"\n",
    "#     if args.local_rank in [-1, 0]:\n",
    "#         tb_writer = SummaryWriter()\n",
    "\n",
    "#     args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu)\n",
    "    train_sampler = RandomSampler(train_dataset) # if args.local_rank == -1 else DistributedSampler(train_dataset)\n",
    "    train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size)\n",
    "\n",
    "    if args.max_steps > 0:\n",
    "        t_total = args.max_steps\n",
    "        args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1\n",
    "    else:\n",
    "        t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs\n",
    "\n",
    "    # Prepare optimizer and schedule (linear warmup and decay)\n",
    "    no_decay = [\"bias\", \"LayerNorm.weight\"]\n",
    "    optimizer_grouped_parameters = [\n",
    "        {\n",
    "            \"params\": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "            \"weight_decay\": args.weight_decay,\n",
    "        },\n",
    "        {\"params\": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], \"weight_decay\": 0.0},\n",
    "    ]\n",
    "\n",
    "    optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)\n",
    "    scheduler = get_linear_schedule_with_warmup(\n",
    "        optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total\n",
    "    )\n",
    "\n",
    "    # Check if saved optimizer or scheduler states exist\n",
    "    if os.path.isfile(os.path.join(args.model_name_or_path, \"optimizer.pt\")) and os.path.isfile(\n",
    "        os.path.join(args.model_name_or_path, \"scheduler.pt\")\n",
    "    ):\n",
    "        logger.info(\"  loading optimizer and scheduler...\")\n",
    "        # Load in optimizer and scheduler states\n",
    "        optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"optimizer.pt\")))\n",
    "#         scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, \"scheduler.pt\")))\n",
    "    else:\n",
    "        logger.info(\"  No optimizer and scheduler,we build a new one\")        \n",
    "\n",
    "    if args.fp16:\n",
    "        try:\n",
    "            from apex import amp\n",
    "        except ImportError:\n",
    "            raise ImportError(\"Please install apex from https://www.github.com/nvidia/apex to use fp16 training.\")\n",
    "        model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)\n",
    "\n",
    "    # multi-gpu training (should be after apex fp16 initialization)\n",
    "    if args.n_gpu > 1:\n",
    "        model = torch.nn.DataParallel(model,device_ids=args.card_list)\n",
    "\n",
    "\n",
    "    # Train!\n",
    "    logger.info(\"***** Running training *****\")\n",
    "    logger.info(\"  Num examples = %d\", len(train_dataset))\n",
    "    logger.info(\"  Num Epochs = %d\", args.num_train_epochs)\n",
    "#     logger.info(\"  Instantaneous batch size per GPU = %d\", args.per_gpu_train_batch_size)\n",
    "    logger.info(\n",
    "        \"  Total train batch size (w. parallel, distributed & accumulation) = %d\",\n",
    "        args.train_batch_size\n",
    "        * args.gradient_accumulation_steps\n",
    "    )\n",
    "    logger.info(\"  Gradient Accumulation steps = %d\", args.gradient_accumulation_steps)\n",
    "    logger.info(\"  Total optimization steps = %d\", t_total)\n",
    "\n",
    "    global_step = 0\n",
    "    epochs_trained = 0\n",
    "    steps_trained_in_current_epoch = 0\n",
    "    # Check if continuing training from a checkpoint\n",
    "    if os.path.exists(args.model_name_or_path):\n",
    "        # set global_step to global_step of last saved checkpoint from model path\n",
    "        try:\n",
    "            global_step = int(args.model_name_or_path.split(\"-\")[-1].split(\"/\")[0])\n",
    "        except ValueError:\n",
    "            global_step = 0\n",
    "        epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "        steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps)\n",
    "\n",
    "        logger.info(\"  Continuing training from checkpoint, will skip to saved global_step\")\n",
    "        logger.info(\"  Continuing training from epoch %d\", epochs_trained)\n",
    "        logger.info(\"  Continuing training from global step %d\", global_step)\n",
    "        logger.info(\"  Will skip the first %d steps in the first epoch\", steps_trained_in_current_epoch)\n",
    "\n",
    "    tr_loss, logging_loss = 0.0, 0.0\n",
    "    model.zero_grad()\n",
    "    train_iterator = tqdm(range(\n",
    "        epochs_trained, int(args.num_train_epochs)), desc=\"Epoch\")\n",
    "    set_seed(args)  # Added here for reproductibility\n",
    "    for _ in train_iterator:\n",
    "        start=time.time()\n",
    "        epoch_iterator = tqdm(train_dataloader, desc=\"Iteration\")\n",
    "\n",
    "        for step, batch in enumerate(epoch_iterator):\n",
    "\n",
    "            # Skip past any already trained steps if resuming training\n",
    "            if steps_trained_in_current_epoch > 0:\n",
    "                if  (step + 1) % args.gradient_accumulation_steps == 0: \n",
    "                        steps_trained_in_current_epoch -= 1\n",
    "                continue\n",
    "\n",
    "            model.train()\n",
    "            tokenvec=batch[6].detach().cpu().numpy()\n",
    "            batch[6]=torch.tensor(word2vec[tokenvec]).float()\n",
    "            wordvec=batch[7].detach().cpu().numpy()\n",
    "            batch[7]=torch.tensor(word2vec[wordvec]).float()\n",
    "            batch = tuple(t.to(args.device) for t in batch[:-1])\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"rel_label\": batch[3],\\\n",
    "                     \"postag\":batch[4],\"feature\":batch[5],\"tokenvec\":batch[6],\"wordvec\":batch[7],\\\n",
    "                      \"wordmask\":batch[8],\"plan_label\":batch[9].float()}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "            outputs = model(**inputs)\n",
    "            loss = outputs[0]  # model outputs are always tuple in transformers (see doc)\n",
    "\n",
    "            if args.n_gpu > 1:\n",
    "                loss = loss.mean()  # mean() to average on multi-gpu parallel training\n",
    "            if args.gradient_accumulation_steps > 1:\n",
    "                loss = loss / args.gradient_accumulation_steps\n",
    "\n",
    "            if args.fp16:\n",
    "                with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "                    scaled_loss.backward()\n",
    "            else:\n",
    "                loss.backward()\n",
    "            logger.info(\"  step:%d loss %.3f\", step,loss.item())\n",
    "\n",
    "            tr_loss += loss.item()\n",
    "            if (step + 1) % args.gradient_accumulation_steps == 0:\n",
    "                if args.fp16:\n",
    "                    torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm)\n",
    "                else:\n",
    "                    torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm)\n",
    "\n",
    "                optimizer.step()\n",
    "                scheduler.step()  # Update learning rate schedule\n",
    "                model.zero_grad()\n",
    "                global_step += 1\n",
    "\n",
    "\n",
    "                if  args.save_steps > 0 and global_step % args.save_steps == 0:\n",
    "                    # Save model checkpoint\n",
    "                    results = evaluate(args, val_dataset,model, tokenizer)\n",
    "                    output_dir = os.path.join(args.output_dir, \"checkpoint-{}\".format(global_step))\n",
    "                    if not os.path.exists(output_dir):\n",
    "                        os.makedirs(output_dir)\n",
    "                    model_to_save = (\n",
    "                        model.module if hasattr(model, \"module\") else model\n",
    "                    )  # Take care of distributed/parallel training\n",
    "                    torch.save(model_to_save.state_dict(),os.path.join(output_dir,\"model.pt\"))\n",
    "                    tokenizer.save_pretrained(output_dir)\n",
    "\n",
    "                    torch.save(args, os.path.join(output_dir, \"training_args.bin\"))\n",
    "                    logger.info(\"Saving model checkpoint to %s\", output_dir)\n",
    "\n",
    "                    torch.save(optimizer.state_dict(), os.path.join(output_dir, \"optimizer.pt\"))\n",
    "                    torch.save(scheduler.state_dict(), os.path.join(output_dir, \"scheduler.pt\"))\n",
    "                    if args.fp16:\n",
    "                        torch.save(amp.state_dict(),os.path.join(output_dir, \"amp.pt\"))\n",
    "                    logger.info(\"Saving optimizer and scheduler states to %s\", output_dir)\n",
    "\n",
    "            if args.max_steps > 0 and global_step > args.max_steps:\n",
    "                epoch_iterator.close()\n",
    "                break\n",
    "        print(time.time()-start)\n",
    "        if args.max_steps > 0 and global_step > args.max_steps:\n",
    "            train_iterator.close()\n",
    "            break\n",
    "\n",
    "\n",
    "    return global_step, tr_loss / global_step\n",
    "def evaluate(args, eval_dataset,model, tokenizer, prefix=\"\"):\n",
    "    eval_output_dir = args.output_dir \n",
    "\n",
    "    results = {}\n",
    "#         eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True)\n",
    "\n",
    "    if not os.path.exists(eval_output_dir) :\n",
    "        os.makedirs(eval_output_dir)\n",
    "\n",
    "#         args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu)\n",
    "    # Note that DistributedSampler samples randomly\n",
    "    eval_sampler = SequentialSampler(eval_dataset)\n",
    "    eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size)\n",
    "\n",
    "    # multi-gpu eval\n",
    "#         if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel):\n",
    "#             model = torch.nn.DataParallel(model)\n",
    "\n",
    "    # Eval!\n",
    "    logger.info(\"***** Running evaluation {} *****\".format(prefix))\n",
    "    logger.info(\"  Num examples = %d\", len(eval_dataset))\n",
    "    logger.info(\"  Batch size = %d\", args.eval_batch_size)\n",
    "    eval_loss = 0.0\n",
    "    nb_eval_steps = 0\n",
    "    preds = None\n",
    "    out_label_ids = None\n",
    "    rights,num=0,0\n",
    "    recall_num,full_num=0,0\n",
    "    precision_num,full_num_pre=0,0\n",
    "    for batch in tqdm(eval_dataloader, desc=\"Evaluating\"):\n",
    "        model.eval()\n",
    "        tokenvec=batch[6].detach().cpu().numpy()\n",
    "        batch[6]=torch.tensor(word2vec[tokenvec]).float()\n",
    "        wordvec=batch[7].detach().cpu().numpy()\n",
    "        batch[7]=torch.tensor(word2vec[wordvec]).float()\n",
    "        batch = tuple(t.to(args.device) for t in batch[:-1])\n",
    "\n",
    "\n",
    "        with torch.no_grad():\n",
    "\n",
    "            inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"rel_label\": batch[3],\\\n",
    "                     \"postag\":batch[4],\"feature\":batch[5],\"tokenvec\":batch[6],\"wordvec\":batch[7],\\\n",
    "                      \"wordmask\":batch[8],\"plan_label\":batch[9].float()}\n",
    "            inputs[\"token_type_ids\"]=batch[2]\n",
    "            outputs = model(**inputs)\n",
    "            tmp_eval_loss, logits = outputs[:2]\n",
    "\n",
    "            eval_loss += tmp_eval_loss.mean().item()\n",
    "        rights+=((logits.detach()>0).long()==batch[3]).all(dim=-1).sum().detach().cpu().item()\n",
    "        num+=batch[3].shape[0]\n",
    "        recall_num+=(((logits.detach()>0).long()==1) & (batch[3]==1)).sum().detach().cpu().item()\n",
    "        precision_num+=(((logits.detach()>0).long()==1) & (batch[3]==1)).sum().detach().cpu().item()\n",
    "        full_num+=batch[3].sum().detach().cpu().item()\n",
    "        full_num_pre+=(logits.detach()>0).long().sum().detach().cpu().item()\n",
    "        nb_eval_steps += 1\n",
    "        if preds is None:\n",
    "            preds = logits.detach().cpu().numpy()\n",
    "            out_label_ids = inputs[\"rel_label\"].detach().cpu().numpy()\n",
    "#             out_label_ids=inputs[\"labels\"].detach().cpu().numpy()\n",
    "        else:\n",
    "            preds = np.append(preds, logits.detach().cpu().numpy(), axis=0)\n",
    "            out_label_ids = np.append(out_label_ids, inputs[\"rel_label\"].detach().cpu().numpy(), axis=0)\n",
    "#             out_label_ids = np.append(out_label_ids, inputs[\"labels\"].detach().cpu().numpy(), axis=0)\n",
    "\n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    if args.output_mode == \"classification\":\n",
    "        preds = (preds>0).astype(np.int8)\n",
    "        out_label_ids=out_label_ids.astype(np.int8)\n",
    "    elif args.output_mode == \"regression\":\n",
    "        preds = np.squeeze(preds)\n",
    "    result = {\"acc_class\":(preds==out_label_ids).sum(axis=0)/preds.shape[0],\"acc_sample\":rights/num,\\\n",
    "              \"recall_sample\":recall_num/full_num,\"precision_sample\":precision_num/(full_num_pre+0.001)}\n",
    "    results.update(result)\n",
    "\n",
    "    output_eval_file = os.path.join(eval_output_dir, prefix, \"eval_results.txt\")\n",
    "    with open(output_eval_file, \"a\") as writer:\n",
    "        logger.info(\"***** Eval results {} *****\".format(prefix))\n",
    "        for key in sorted(result.keys()):\n",
    "            logger.info(\"  %s = %s\", key, str(result[key]))\n",
    "            writer.write(\"%s = %s\\n\" % (key, str(result[key])))\n",
    "\n",
    "    return results,preds,out_label_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "code_folding": [],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:55:00 - INFO - transformers.configuration_utils -   loading configuration file ./roberta-zh-wwm-pytorch/bert_config.json\n",
      "05/05/2020 21:55:00 - INFO - transformers.configuration_utils -   Model config BertConfig {\n",
      "  \"_num_labels\": 2,\n",
      "  \"architectures\": null,\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": null,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"do_sample\": false,\n",
      "  \"early_stopping\": false,\n",
      "  \"eos_token_ids\": null,\n",
      "  \"finetuning_task\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"is_decoder\": false,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"length_penalty\": 1.0,\n",
      "  \"max_length\": 20,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_beams\": 1,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"num_return_sequences\": 1,\n",
      "  \"output_attentions\": true,\n",
      "  \"output_hidden_states\": true,\n",
      "  \"output_past\": true,\n",
      "  \"pad_token_id\": null,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"pruned_heads\": {},\n",
      "  \"repetition_penalty\": 1.0,\n",
      "  \"temperature\": 1.0,\n",
      "  \"top_k\": 50,\n",
      "  \"top_p\": 1.0,\n",
      "  \"torchscript\": false,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_bfloat16\": false,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "05/05/2020 21:55:00 - INFO - transformers.modeling_utils -   loading weights file ./roberta-zh-wwm-pytorch/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "model=BertMulticlass_DGCNN(bert_dir,config_file,num_rel_labels=num_rel_labels,\n",
    "                  hidden_dropout_prob=hidden_dropout_prob,embed_dim=128,loss_type=True,use_feature=True,use_vec=False,use_word=True,use_pool=True,use_plan=True,alpha=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:10:53 - INFO - __main__ -     No optimizer and scheduler,we build a new one\n",
      "05/05/2020 21:10:53 - INFO - __main__ -   ***** Running training *****\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Num examples = 10000\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Num Epochs = 3\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Total train batch size (w. parallel, distributed & accumulation) = 32\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Gradient Accumulation steps = 1\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Total optimization steps = 939\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Continuing training from checkpoint, will skip to saved global_step\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Continuing training from epoch 0\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Continuing training from global step 0\n",
      "05/05/2020 21:10:53 - INFO - __main__ -     Will skip the first 0 steps in the first epoch\n",
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:309: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4c71fe0c48e46218f2883de706b5503",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=3, style=ProgressStyle(description_width='initial…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:313: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a2f31380de64ca4b2a3c46954c1f4ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=313, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/torch/nn/parallel/_functions.py:61: UserWarning: Was asked to gather along dimension 0, but all input tensors were scalars; will instead unsqueeze and return a vector.\n",
      "  warnings.warn('Was asked to gather along dimension 0, but all '\n",
      "05/05/2020 21:10:59 - INFO - __main__ -     step:0 loss 1.372\n",
      "05/05/2020 21:11:00 - INFO - __main__ -     step:1 loss 1.348\n",
      "05/05/2020 21:11:01 - INFO - __main__ -     step:2 loss 1.327\n",
      "05/05/2020 21:11:02 - INFO - __main__ -     step:3 loss 1.261\n",
      "05/05/2020 21:11:03 - INFO - __main__ -     step:4 loss 1.292\n",
      "05/05/2020 21:11:04 - INFO - __main__ -     step:5 loss 1.275\n",
      "05/05/2020 21:11:06 - INFO - __main__ -     step:6 loss 1.232\n",
      "05/05/2020 21:11:07 - INFO - __main__ -     step:7 loss 1.194\n",
      "05/05/2020 21:11:08 - INFO - __main__ -     step:8 loss 1.158\n",
      "05/05/2020 21:11:09 - INFO - __main__ -     step:9 loss 1.130\n",
      "05/05/2020 21:11:10 - INFO - __main__ -     step:10 loss 1.048\n",
      "05/05/2020 21:11:11 - INFO - __main__ -     step:11 loss 1.130\n",
      "05/05/2020 21:11:12 - INFO - __main__ -     step:12 loss 1.090\n",
      "05/05/2020 21:11:14 - INFO - __main__ -     step:13 loss 1.060\n",
      "05/05/2020 21:11:15 - INFO - __main__ -     step:14 loss 0.987\n",
      "05/05/2020 21:11:16 - INFO - __main__ -     step:15 loss 1.094\n",
      "05/05/2020 21:11:17 - INFO - __main__ -     step:16 loss 0.956\n",
      "05/05/2020 21:11:18 - INFO - __main__ -     step:17 loss 1.047\n",
      "05/05/2020 21:11:19 - INFO - __main__ -     step:18 loss 0.986\n",
      "05/05/2020 21:11:20 - INFO - __main__ -     step:19 loss 0.898\n",
      "05/05/2020 21:11:22 - INFO - __main__ -     step:20 loss 0.920\n",
      "05/05/2020 21:11:23 - INFO - __main__ -     step:21 loss 0.871\n",
      "05/05/2020 21:11:24 - INFO - __main__ -     step:22 loss 0.937\n",
      "05/05/2020 21:11:25 - INFO - __main__ -     step:23 loss 0.840\n",
      "05/05/2020 21:11:26 - INFO - __main__ -     step:24 loss 0.871\n",
      "05/05/2020 21:11:27 - INFO - __main__ -     step:25 loss 0.822\n",
      "05/05/2020 21:11:29 - INFO - __main__ -     step:26 loss 0.817\n",
      "05/05/2020 21:11:30 - INFO - __main__ -     step:27 loss 0.855\n",
      "05/05/2020 21:11:31 - INFO - __main__ -     step:28 loss 0.802\n",
      "05/05/2020 21:11:32 - INFO - __main__ -     step:29 loss 0.858\n",
      "05/05/2020 21:11:33 - INFO - __main__ -     step:30 loss 0.798\n",
      "05/05/2020 21:11:34 - INFO - __main__ -     step:31 loss 0.787\n",
      "05/05/2020 21:11:35 - INFO - __main__ -     step:32 loss 0.689\n",
      "05/05/2020 21:11:37 - INFO - __main__ -     step:33 loss 0.838\n",
      "05/05/2020 21:11:38 - INFO - __main__ -     step:34 loss 0.676\n",
      "05/05/2020 21:11:39 - INFO - __main__ -     step:35 loss 0.707\n",
      "05/05/2020 21:11:40 - INFO - __main__ -     step:36 loss 0.698\n",
      "05/05/2020 21:11:41 - INFO - __main__ -     step:37 loss 0.715\n",
      "05/05/2020 21:11:42 - INFO - __main__ -     step:38 loss 0.802\n",
      "05/05/2020 21:11:44 - INFO - __main__ -     step:39 loss 0.684\n",
      "05/05/2020 21:11:45 - INFO - __main__ -     step:40 loss 0.647\n",
      "05/05/2020 21:11:46 - INFO - __main__ -     step:41 loss 0.637\n",
      "05/05/2020 21:11:47 - INFO - __main__ -     step:42 loss 0.707\n",
      "05/05/2020 21:11:48 - INFO - __main__ -     step:43 loss 0.626\n",
      "05/05/2020 21:11:49 - INFO - __main__ -     step:44 loss 0.607\n",
      "05/05/2020 21:11:51 - INFO - __main__ -     step:45 loss 0.724\n",
      "05/05/2020 21:11:52 - INFO - __main__ -     step:46 loss 0.601\n",
      "05/05/2020 21:11:53 - INFO - __main__ -     step:47 loss 0.635\n",
      "05/05/2020 21:11:54 - INFO - __main__ -     step:48 loss 0.604\n",
      "05/05/2020 21:11:55 - INFO - __main__ -     step:49 loss 0.658\n",
      "05/05/2020 21:11:57 - INFO - __main__ -     step:50 loss 0.663\n",
      "05/05/2020 21:11:58 - INFO - __main__ -     step:51 loss 0.550\n",
      "05/05/2020 21:11:59 - INFO - __main__ -     step:52 loss 0.584\n",
      "05/05/2020 21:12:00 - INFO - __main__ -     step:53 loss 0.513\n",
      "05/05/2020 21:12:02 - INFO - __main__ -     step:54 loss 0.620\n",
      "05/05/2020 21:12:03 - INFO - __main__ -     step:55 loss 0.541\n",
      "05/05/2020 21:12:04 - INFO - __main__ -     step:56 loss 0.580\n",
      "05/05/2020 21:12:05 - INFO - __main__ -     step:57 loss 0.582\n",
      "05/05/2020 21:12:06 - INFO - __main__ -     step:58 loss 0.470\n",
      "05/05/2020 21:12:08 - INFO - __main__ -     step:59 loss 0.434\n",
      "05/05/2020 21:12:09 - INFO - __main__ -     step:60 loss 0.574\n",
      "05/05/2020 21:12:10 - INFO - __main__ -     step:61 loss 0.470\n",
      "05/05/2020 21:12:11 - INFO - __main__ -     step:62 loss 0.470\n",
      "05/05/2020 21:12:13 - INFO - __main__ -     step:63 loss 0.408\n",
      "05/05/2020 21:12:14 - INFO - __main__ -     step:64 loss 0.429\n",
      "05/05/2020 21:12:15 - INFO - __main__ -     step:65 loss 0.409\n",
      "05/05/2020 21:12:16 - INFO - __main__ -     step:66 loss 0.503\n",
      "05/05/2020 21:12:17 - INFO - __main__ -     step:67 loss 0.472\n",
      "05/05/2020 21:12:19 - INFO - __main__ -     step:68 loss 0.472\n",
      "05/05/2020 21:12:20 - INFO - __main__ -     step:69 loss 0.534\n",
      "05/05/2020 21:12:21 - INFO - __main__ -     step:70 loss 0.418\n",
      "05/05/2020 21:12:22 - INFO - __main__ -     step:71 loss 0.511\n",
      "05/05/2020 21:12:24 - INFO - __main__ -     step:72 loss 0.432\n",
      "05/05/2020 21:12:25 - INFO - __main__ -     step:73 loss 0.514\n",
      "05/05/2020 21:12:26 - INFO - __main__ -     step:74 loss 0.377\n",
      "05/05/2020 21:12:27 - INFO - __main__ -     step:75 loss 0.391\n",
      "05/05/2020 21:12:29 - INFO - __main__ -     step:76 loss 0.387\n",
      "05/05/2020 21:12:30 - INFO - __main__ -     step:77 loss 0.374\n",
      "05/05/2020 21:12:31 - INFO - __main__ -     step:78 loss 0.413\n",
      "05/05/2020 21:12:32 - INFO - __main__ -     step:79 loss 0.444\n",
      "05/05/2020 21:12:34 - INFO - __main__ -     step:80 loss 0.441\n",
      "05/05/2020 21:12:35 - INFO - __main__ -     step:81 loss 0.400\n",
      "05/05/2020 21:12:36 - INFO - __main__ -     step:82 loss 0.363\n",
      "05/05/2020 21:12:37 - INFO - __main__ -     step:83 loss 0.492\n",
      "05/05/2020 21:12:39 - INFO - __main__ -     step:84 loss 0.461\n",
      "05/05/2020 21:12:40 - INFO - __main__ -     step:85 loss 0.435\n",
      "05/05/2020 21:12:41 - INFO - __main__ -     step:86 loss 0.525\n",
      "05/05/2020 21:12:42 - INFO - __main__ -     step:87 loss 0.521\n",
      "05/05/2020 21:12:44 - INFO - __main__ -     step:88 loss 0.414\n",
      "05/05/2020 21:12:45 - INFO - __main__ -     step:89 loss 0.432\n",
      "05/05/2020 21:12:46 - INFO - __main__ -     step:90 loss 0.486\n",
      "05/05/2020 21:12:47 - INFO - __main__ -     step:91 loss 0.386\n",
      "05/05/2020 21:12:49 - INFO - __main__ -     step:92 loss 0.418\n",
      "05/05/2020 21:12:50 - INFO - __main__ -     step:93 loss 0.455\n",
      "05/05/2020 21:12:51 - INFO - __main__ -     step:94 loss 0.352\n",
      "05/05/2020 21:12:52 - INFO - __main__ -     step:95 loss 0.332\n",
      "05/05/2020 21:12:54 - INFO - __main__ -     step:96 loss 0.358\n",
      "05/05/2020 21:12:55 - INFO - __main__ -     step:97 loss 0.507\n",
      "05/05/2020 21:12:56 - INFO - __main__ -     step:98 loss 0.356\n",
      "05/05/2020 21:12:57 - INFO - __main__ -     step:99 loss 0.391\n",
      "05/05/2020 21:12:59 - INFO - __main__ -     step:100 loss 0.383\n",
      "05/05/2020 21:13:00 - INFO - __main__ -     step:101 loss 0.360\n",
      "05/05/2020 21:13:01 - INFO - __main__ -     step:102 loss 0.343\n",
      "05/05/2020 21:13:02 - INFO - __main__ -     step:103 loss 0.363\n",
      "05/05/2020 21:13:04 - INFO - __main__ -     step:104 loss 0.352\n",
      "05/05/2020 21:13:05 - INFO - __main__ -     step:105 loss 0.368\n",
      "05/05/2020 21:13:06 - INFO - __main__ -     step:106 loss 0.353\n",
      "05/05/2020 21:13:07 - INFO - __main__ -     step:107 loss 0.308\n",
      "05/05/2020 21:13:09 - INFO - __main__ -     step:108 loss 0.395\n",
      "05/05/2020 21:13:10 - INFO - __main__ -     step:109 loss 0.367\n",
      "05/05/2020 21:13:11 - INFO - __main__ -     step:110 loss 0.364\n",
      "05/05/2020 21:13:12 - INFO - __main__ -     step:111 loss 0.270\n",
      "05/05/2020 21:13:14 - INFO - __main__ -     step:112 loss 0.412\n",
      "05/05/2020 21:13:15 - INFO - __main__ -     step:113 loss 0.357\n",
      "05/05/2020 21:13:16 - INFO - __main__ -     step:114 loss 0.416\n",
      "05/05/2020 21:13:18 - INFO - __main__ -     step:115 loss 0.359\n",
      "05/05/2020 21:13:19 - INFO - __main__ -     step:116 loss 0.291\n",
      "05/05/2020 21:13:20 - INFO - __main__ -     step:117 loss 0.409\n",
      "05/05/2020 21:13:21 - INFO - __main__ -     step:118 loss 0.414\n",
      "05/05/2020 21:13:23 - INFO - __main__ -     step:119 loss 0.391\n",
      "05/05/2020 21:13:24 - INFO - __main__ -     step:120 loss 0.369\n",
      "05/05/2020 21:13:25 - INFO - __main__ -     step:121 loss 0.318\n",
      "05/05/2020 21:13:26 - INFO - __main__ -     step:122 loss 0.362\n",
      "05/05/2020 21:13:28 - INFO - __main__ -     step:123 loss 0.320\n",
      "05/05/2020 21:13:29 - INFO - __main__ -     step:124 loss 0.403\n",
      "05/05/2020 21:13:30 - INFO - __main__ -     step:125 loss 0.308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:13:31 - INFO - __main__ -     step:126 loss 0.370\n",
      "05/05/2020 21:13:33 - INFO - __main__ -     step:127 loss 0.409\n",
      "05/05/2020 21:13:34 - INFO - __main__ -     step:128 loss 0.425\n",
      "05/05/2020 21:13:35 - INFO - __main__ -     step:129 loss 0.394\n",
      "05/05/2020 21:13:36 - INFO - __main__ -     step:130 loss 0.451\n",
      "05/05/2020 21:13:38 - INFO - __main__ -     step:131 loss 0.252\n",
      "05/05/2020 21:13:39 - INFO - __main__ -     step:132 loss 0.413\n",
      "05/05/2020 21:13:40 - INFO - __main__ -     step:133 loss 0.252\n",
      "05/05/2020 21:13:41 - INFO - __main__ -     step:134 loss 0.375\n",
      "05/05/2020 21:13:43 - INFO - __main__ -     step:135 loss 0.265\n",
      "05/05/2020 21:13:44 - INFO - __main__ -     step:136 loss 0.280\n",
      "05/05/2020 21:13:45 - INFO - __main__ -     step:137 loss 0.347\n",
      "05/05/2020 21:13:47 - INFO - __main__ -     step:138 loss 0.291\n",
      "05/05/2020 21:13:48 - INFO - __main__ -     step:139 loss 0.300\n",
      "05/05/2020 21:13:49 - INFO - __main__ -     step:140 loss 0.349\n",
      "05/05/2020 21:13:50 - INFO - __main__ -     step:141 loss 0.363\n",
      "05/05/2020 21:13:52 - INFO - __main__ -     step:142 loss 0.389\n",
      "05/05/2020 21:13:53 - INFO - __main__ -     step:143 loss 0.262\n",
      "05/05/2020 21:13:54 - INFO - __main__ -     step:144 loss 0.378\n",
      "05/05/2020 21:13:55 - INFO - __main__ -     step:145 loss 0.310\n",
      "05/05/2020 21:13:57 - INFO - __main__ -     step:146 loss 0.348\n",
      "05/05/2020 21:13:58 - INFO - __main__ -     step:147 loss 0.393\n",
      "05/05/2020 21:13:59 - INFO - __main__ -     step:148 loss 0.371\n",
      "05/05/2020 21:14:00 - INFO - __main__ -     step:149 loss 0.282\n",
      "05/05/2020 21:14:02 - INFO - __main__ -     step:150 loss 0.269\n",
      "05/05/2020 21:14:03 - INFO - __main__ -     step:151 loss 0.412\n",
      "05/05/2020 21:14:04 - INFO - __main__ -     step:152 loss 0.349\n",
      "05/05/2020 21:14:05 - INFO - __main__ -     step:153 loss 0.400\n",
      "05/05/2020 21:14:07 - INFO - __main__ -     step:154 loss 0.297\n",
      "05/05/2020 21:14:08 - INFO - __main__ -     step:155 loss 0.284\n",
      "05/05/2020 21:14:09 - INFO - __main__ -     step:156 loss 0.272\n",
      "05/05/2020 21:14:10 - INFO - __main__ -     step:157 loss 0.246\n",
      "05/05/2020 21:14:12 - INFO - __main__ -     step:158 loss 0.339\n",
      "05/05/2020 21:14:13 - INFO - __main__ -     step:159 loss 0.310\n",
      "05/05/2020 21:14:14 - INFO - __main__ -     step:160 loss 0.170\n",
      "05/05/2020 21:14:15 - INFO - __main__ -     step:161 loss 0.295\n",
      "05/05/2020 21:14:17 - INFO - __main__ -     step:162 loss 0.334\n",
      "05/05/2020 21:14:18 - INFO - __main__ -     step:163 loss 0.430\n",
      "05/05/2020 21:14:19 - INFO - __main__ -     step:164 loss 0.276\n",
      "05/05/2020 21:14:20 - INFO - __main__ -     step:165 loss 0.379\n",
      "05/05/2020 21:14:22 - INFO - __main__ -     step:166 loss 0.264\n",
      "05/05/2020 21:14:23 - INFO - __main__ -     step:167 loss 0.362\n",
      "05/05/2020 21:14:24 - INFO - __main__ -     step:168 loss 0.273\n",
      "05/05/2020 21:14:25 - INFO - __main__ -     step:169 loss 0.367\n",
      "05/05/2020 21:14:27 - INFO - __main__ -     step:170 loss 0.312\n",
      "05/05/2020 21:14:28 - INFO - __main__ -     step:171 loss 0.275\n",
      "05/05/2020 21:14:29 - INFO - __main__ -     step:172 loss 0.271\n",
      "05/05/2020 21:14:30 - INFO - __main__ -     step:173 loss 0.305\n",
      "05/05/2020 21:14:32 - INFO - __main__ -     step:174 loss 0.269\n",
      "05/05/2020 21:14:33 - INFO - __main__ -     step:175 loss 0.358\n",
      "05/05/2020 21:14:34 - INFO - __main__ -     step:176 loss 0.257\n",
      "05/05/2020 21:14:36 - INFO - __main__ -     step:177 loss 0.214\n",
      "05/05/2020 21:14:37 - INFO - __main__ -     step:178 loss 0.336\n",
      "05/05/2020 21:14:38 - INFO - __main__ -     step:179 loss 0.307\n",
      "05/05/2020 21:14:40 - INFO - __main__ -     step:180 loss 0.358\n",
      "05/05/2020 21:14:41 - INFO - __main__ -     step:181 loss 0.287\n",
      "05/05/2020 21:14:42 - INFO - __main__ -     step:182 loss 0.197\n",
      "05/05/2020 21:14:43 - INFO - __main__ -     step:183 loss 0.194\n",
      "05/05/2020 21:14:45 - INFO - __main__ -     step:184 loss 0.285\n",
      "05/05/2020 21:14:46 - INFO - __main__ -     step:185 loss 0.272\n",
      "05/05/2020 21:14:47 - INFO - __main__ -     step:186 loss 0.455\n",
      "05/05/2020 21:14:48 - INFO - __main__ -     step:187 loss 0.395\n",
      "05/05/2020 21:14:50 - INFO - __main__ -     step:188 loss 0.298\n",
      "05/05/2020 21:14:51 - INFO - __main__ -     step:189 loss 0.320\n",
      "05/05/2020 21:14:52 - INFO - __main__ -     step:190 loss 0.324\n",
      "05/05/2020 21:14:53 - INFO - __main__ -     step:191 loss 0.254\n",
      "05/05/2020 21:14:55 - INFO - __main__ -     step:192 loss 0.206\n",
      "05/05/2020 21:14:56 - INFO - __main__ -     step:193 loss 0.374\n",
      "05/05/2020 21:14:57 - INFO - __main__ -     step:194 loss 0.278\n",
      "05/05/2020 21:14:59 - INFO - __main__ -     step:195 loss 0.289\n",
      "05/05/2020 21:15:00 - INFO - __main__ -     step:196 loss 0.265\n",
      "05/05/2020 21:15:01 - INFO - __main__ -     step:197 loss 0.301\n",
      "05/05/2020 21:15:02 - INFO - __main__ -     step:198 loss 0.287\n",
      "05/05/2020 21:15:04 - INFO - __main__ -     step:199 loss 0.172\n",
      "05/05/2020 21:15:05 - INFO - __main__ -     step:200 loss 0.290\n",
      "05/05/2020 21:15:06 - INFO - __main__ -     step:201 loss 0.319\n",
      "05/05/2020 21:15:08 - INFO - __main__ -     step:202 loss 0.280\n",
      "05/05/2020 21:15:09 - INFO - __main__ -     step:203 loss 0.294\n",
      "05/05/2020 21:15:10 - INFO - __main__ -     step:204 loss 0.362\n",
      "05/05/2020 21:15:12 - INFO - __main__ -     step:205 loss 0.264\n",
      "05/05/2020 21:15:13 - INFO - __main__ -     step:206 loss 0.253\n",
      "05/05/2020 21:15:14 - INFO - __main__ -     step:207 loss 0.271\n",
      "05/05/2020 21:15:15 - INFO - __main__ -     step:208 loss 0.211\n",
      "05/05/2020 21:15:17 - INFO - __main__ -     step:209 loss 0.373\n",
      "05/05/2020 21:15:18 - INFO - __main__ -     step:210 loss 0.270\n",
      "05/05/2020 21:15:19 - INFO - __main__ -     step:211 loss 0.343\n",
      "05/05/2020 21:15:20 - INFO - __main__ -     step:212 loss 0.333\n",
      "05/05/2020 21:15:22 - INFO - __main__ -     step:213 loss 0.204\n",
      "05/05/2020 21:15:23 - INFO - __main__ -     step:214 loss 0.279\n",
      "05/05/2020 21:15:24 - INFO - __main__ -     step:215 loss 0.241\n",
      "05/05/2020 21:15:26 - INFO - __main__ -     step:216 loss 0.425\n",
      "05/05/2020 21:15:27 - INFO - __main__ -     step:217 loss 0.269\n",
      "05/05/2020 21:15:28 - INFO - __main__ -     step:218 loss 0.293\n",
      "05/05/2020 21:15:29 - INFO - __main__ -     step:219 loss 0.358\n",
      "05/05/2020 21:15:31 - INFO - __main__ -     step:220 loss 0.250\n",
      "05/05/2020 21:15:32 - INFO - __main__ -     step:221 loss 0.382\n",
      "05/05/2020 21:15:33 - INFO - __main__ -     step:222 loss 0.266\n",
      "05/05/2020 21:15:34 - INFO - __main__ -     step:223 loss 0.278\n",
      "05/05/2020 21:15:36 - INFO - __main__ -     step:224 loss 0.307\n",
      "05/05/2020 21:15:37 - INFO - __main__ -     step:225 loss 0.298\n",
      "05/05/2020 21:15:38 - INFO - __main__ -     step:226 loss 0.260\n",
      "05/05/2020 21:15:40 - INFO - __main__ -     step:227 loss 0.296\n",
      "05/05/2020 21:15:41 - INFO - __main__ -     step:228 loss 0.263\n",
      "05/05/2020 21:15:42 - INFO - __main__ -     step:229 loss 0.317\n",
      "05/05/2020 21:15:43 - INFO - __main__ -     step:230 loss 0.181\n",
      "05/05/2020 21:15:45 - INFO - __main__ -     step:231 loss 0.216\n",
      "05/05/2020 21:15:46 - INFO - __main__ -     step:232 loss 0.287\n",
      "05/05/2020 21:15:47 - INFO - __main__ -     step:233 loss 0.306\n",
      "05/05/2020 21:15:49 - INFO - __main__ -     step:234 loss 0.294\n",
      "05/05/2020 21:15:50 - INFO - __main__ -     step:235 loss 0.252\n",
      "05/05/2020 21:15:51 - INFO - __main__ -     step:236 loss 0.231\n",
      "05/05/2020 21:15:52 - INFO - __main__ -     step:237 loss 0.295\n",
      "05/05/2020 21:15:54 - INFO - __main__ -     step:238 loss 0.269\n",
      "05/05/2020 21:15:55 - INFO - __main__ -     step:239 loss 0.395\n",
      "05/05/2020 21:15:56 - INFO - __main__ -     step:240 loss 0.226\n",
      "05/05/2020 21:15:58 - INFO - __main__ -     step:241 loss 0.276\n",
      "05/05/2020 21:15:59 - INFO - __main__ -     step:242 loss 0.237\n",
      "05/05/2020 21:16:00 - INFO - __main__ -     step:243 loss 0.187\n",
      "05/05/2020 21:16:01 - INFO - __main__ -     step:244 loss 0.235\n",
      "05/05/2020 21:16:03 - INFO - __main__ -     step:245 loss 0.232\n",
      "05/05/2020 21:16:04 - INFO - __main__ -     step:246 loss 0.185\n",
      "05/05/2020 21:16:05 - INFO - __main__ -     step:247 loss 0.248\n",
      "05/05/2020 21:16:06 - INFO - __main__ -     step:248 loss 0.214\n",
      "05/05/2020 21:16:08 - INFO - __main__ -     step:249 loss 0.335\n",
      "05/05/2020 21:16:09 - INFO - __main__ -     step:250 loss 0.282\n",
      "05/05/2020 21:16:10 - INFO - __main__ -     step:251 loss 0.316\n",
      "05/05/2020 21:16:12 - INFO - __main__ -     step:252 loss 0.228\n",
      "05/05/2020 21:16:13 - INFO - __main__ -     step:253 loss 0.185\n",
      "05/05/2020 21:16:14 - INFO - __main__ -     step:254 loss 0.242\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:16:15 - INFO - __main__ -     step:255 loss 0.238\n",
      "05/05/2020 21:16:17 - INFO - __main__ -     step:256 loss 0.220\n",
      "05/05/2020 21:16:18 - INFO - __main__ -     step:257 loss 0.170\n",
      "05/05/2020 21:16:19 - INFO - __main__ -     step:258 loss 0.202\n",
      "05/05/2020 21:16:20 - INFO - __main__ -     step:259 loss 0.183\n",
      "05/05/2020 21:16:22 - INFO - __main__ -     step:260 loss 0.175\n",
      "05/05/2020 21:16:23 - INFO - __main__ -     step:261 loss 0.247\n",
      "05/05/2020 21:16:24 - INFO - __main__ -     step:262 loss 0.258\n",
      "05/05/2020 21:16:26 - INFO - __main__ -     step:263 loss 0.328\n",
      "05/05/2020 21:16:27 - INFO - __main__ -     step:264 loss 0.364\n",
      "05/05/2020 21:16:28 - INFO - __main__ -     step:265 loss 0.311\n",
      "05/05/2020 21:16:30 - INFO - __main__ -     step:266 loss 0.181\n",
      "05/05/2020 21:16:31 - INFO - __main__ -     step:267 loss 0.222\n",
      "05/05/2020 21:16:32 - INFO - __main__ -     step:268 loss 0.265\n",
      "05/05/2020 21:16:34 - INFO - __main__ -     step:269 loss 0.238\n",
      "05/05/2020 21:16:35 - INFO - __main__ -     step:270 loss 0.171\n",
      "05/05/2020 21:16:36 - INFO - __main__ -     step:271 loss 0.201\n",
      "05/05/2020 21:16:37 - INFO - __main__ -     step:272 loss 0.153\n",
      "05/05/2020 21:16:39 - INFO - __main__ -     step:273 loss 0.276\n",
      "05/05/2020 21:16:40 - INFO - __main__ -     step:274 loss 0.320\n",
      "05/05/2020 21:16:41 - INFO - __main__ -     step:275 loss 0.280\n",
      "05/05/2020 21:16:42 - INFO - __main__ -     step:276 loss 0.179\n",
      "05/05/2020 21:16:44 - INFO - __main__ -     step:277 loss 0.193\n",
      "05/05/2020 21:16:45 - INFO - __main__ -     step:278 loss 0.182\n",
      "05/05/2020 21:16:46 - INFO - __main__ -     step:279 loss 0.230\n",
      "05/05/2020 21:16:48 - INFO - __main__ -     step:280 loss 0.250\n",
      "05/05/2020 21:16:49 - INFO - __main__ -     step:281 loss 0.157\n",
      "05/05/2020 21:16:50 - INFO - __main__ -     step:282 loss 0.175\n",
      "05/05/2020 21:16:51 - INFO - __main__ -     step:283 loss 0.203\n",
      "05/05/2020 21:16:53 - INFO - __main__ -     step:284 loss 0.181\n",
      "05/05/2020 21:16:54 - INFO - __main__ -     step:285 loss 0.183\n",
      "05/05/2020 21:16:55 - INFO - __main__ -     step:286 loss 0.321\n",
      "05/05/2020 21:16:57 - INFO - __main__ -     step:287 loss 0.290\n",
      "05/05/2020 21:16:58 - INFO - __main__ -     step:288 loss 0.164\n",
      "05/05/2020 21:16:59 - INFO - __main__ -     step:289 loss 0.187\n",
      "05/05/2020 21:17:00 - INFO - __main__ -     step:290 loss 0.275\n",
      "05/05/2020 21:17:02 - INFO - __main__ -     step:291 loss 0.282\n",
      "05/05/2020 21:17:03 - INFO - __main__ -     step:292 loss 0.298\n",
      "05/05/2020 21:17:04 - INFO - __main__ -     step:293 loss 0.204\n",
      "05/05/2020 21:17:06 - INFO - __main__ -     step:294 loss 0.278\n",
      "05/05/2020 21:17:07 - INFO - __main__ -     step:295 loss 0.222\n",
      "05/05/2020 21:17:08 - INFO - __main__ -     step:296 loss 0.266\n",
      "05/05/2020 21:17:10 - INFO - __main__ -     step:297 loss 0.256\n",
      "05/05/2020 21:17:11 - INFO - __main__ -     step:298 loss 0.226\n",
      "05/05/2020 21:17:12 - INFO - __main__ -     step:299 loss 0.253\n",
      "05/05/2020 21:17:13 - INFO - __main__ -     step:300 loss 0.251\n",
      "05/05/2020 21:17:14 - INFO - __main__ -     step:301 loss 0.275\n",
      "05/05/2020 21:17:16 - INFO - __main__ -     step:302 loss 0.219\n",
      "05/05/2020 21:17:17 - INFO - __main__ -     step:303 loss 0.211\n",
      "05/05/2020 21:17:18 - INFO - __main__ -     step:304 loss 0.256\n",
      "05/05/2020 21:17:20 - INFO - __main__ -     step:305 loss 0.215\n",
      "05/05/2020 21:17:21 - INFO - __main__ -     step:306 loss 0.357\n",
      "05/05/2020 21:17:22 - INFO - __main__ -     step:307 loss 0.223\n",
      "05/05/2020 21:17:24 - INFO - __main__ -     step:308 loss 0.244\n",
      "05/05/2020 21:17:25 - INFO - __main__ -     step:309 loss 0.214\n",
      "05/05/2020 21:17:26 - INFO - __main__ -     step:310 loss 0.192\n",
      "05/05/2020 21:17:28 - INFO - __main__ -     step:311 loss 0.219\n",
      "05/05/2020 21:17:28 - INFO - __main__ -     step:312 loss 0.213\n",
      "05/05/2020 21:17:28 - INFO - __main__ -   ***** Running evaluation  *****\n",
      "05/05/2020 21:17:28 - INFO - __main__ -     Num examples = 1000\n",
      "05/05/2020 21:17:28 - INFO - __main__ -     Batch size = 32\n",
      "/home/huangweilin/anaconda3/envs/fjw/lib/python3.6/site-packages/ipykernel_launcher.py:421: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a22a2d0363a44fda9c70bff020bdb31c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=32, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:17:39 - INFO - __main__ -   ***** Eval results  *****\n",
      "05/05/2020 21:17:39 - INFO - __main__ -     acc_class = [0.95  0.987 0.995 0.982 0.881 1.    0.98  0.99  0.987 0.916 0.874 0.985\n",
      " 0.985 0.898 0.984 0.991 0.928 0.948 0.894 0.995 0.89  0.978 0.944 0.946\n",
      " 0.992 0.96  0.891 0.988 0.923 0.861 0.964 0.998 0.995 1.    0.894 0.883\n",
      " 0.962 0.983 0.946 0.997 0.991 0.984 0.92  0.956 1.    0.92  0.929 0.99\n",
      " 0.956 0.96  1.    0.94  0.976 0.716 0.984]\n",
      "05/05/2020 21:17:39 - INFO - __main__ -     acc_sample = 0.066\n",
      "05/05/2020 21:17:39 - INFO - __main__ -     precision_sample = 0.368836200977268\n",
      "05/05/2020 21:17:39 - INFO - __main__ -     recall_sample = 0.9534735500318674\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:17:42 - INFO - __main__ -   Saving model checkpoint to ./output_test/checkpoint-313\n",
      "05/05/2020 21:17:52 - INFO - __main__ -   Saving optimizer and scheduler states to ./output_test/checkpoint-313\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "418.6611132621765\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86c221e2240b47d58d30332e2e20c03d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=313, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:17:53 - INFO - __main__ -     step:0 loss 0.159\n",
      "05/05/2020 21:17:54 - INFO - __main__ -     step:1 loss 0.163\n",
      "05/05/2020 21:17:55 - INFO - __main__ -     step:2 loss 0.175\n",
      "05/05/2020 21:17:57 - INFO - __main__ -     step:3 loss 0.203\n",
      "05/05/2020 21:17:58 - INFO - __main__ -     step:4 loss 0.194\n",
      "05/05/2020 21:17:59 - INFO - __main__ -     step:5 loss 0.177\n",
      "05/05/2020 21:18:00 - INFO - __main__ -     step:6 loss 0.158\n",
      "05/05/2020 21:18:01 - INFO - __main__ -     step:7 loss 0.137\n",
      "05/05/2020 21:18:02 - INFO - __main__ -     step:8 loss 0.232\n",
      "05/05/2020 21:18:04 - INFO - __main__ -     step:9 loss 0.128\n",
      "05/05/2020 21:18:05 - INFO - __main__ -     step:10 loss 0.193\n",
      "05/05/2020 21:18:06 - INFO - __main__ -     step:11 loss 0.139\n",
      "05/05/2020 21:18:07 - INFO - __main__ -     step:12 loss 0.200\n",
      "05/05/2020 21:18:08 - INFO - __main__ -     step:13 loss 0.151\n",
      "05/05/2020 21:18:10 - INFO - __main__ -     step:14 loss 0.246\n",
      "05/05/2020 21:18:11 - INFO - __main__ -     step:15 loss 0.192\n",
      "05/05/2020 21:18:12 - INFO - __main__ -     step:16 loss 0.193\n",
      "05/05/2020 21:18:13 - INFO - __main__ -     step:17 loss 0.149\n",
      "05/05/2020 21:18:15 - INFO - __main__ -     step:18 loss 0.176\n",
      "05/05/2020 21:18:16 - INFO - __main__ -     step:19 loss 0.139\n",
      "05/05/2020 21:18:17 - INFO - __main__ -     step:20 loss 0.170\n",
      "05/05/2020 21:18:19 - INFO - __main__ -     step:21 loss 0.182\n",
      "05/05/2020 21:18:20 - INFO - __main__ -     step:22 loss 0.139\n",
      "05/05/2020 21:18:21 - INFO - __main__ -     step:23 loss 0.177\n",
      "05/05/2020 21:18:22 - INFO - __main__ -     step:24 loss 0.164\n",
      "05/05/2020 21:18:24 - INFO - __main__ -     step:25 loss 0.189\n",
      "05/05/2020 21:18:25 - INFO - __main__ -     step:26 loss 0.143\n",
      "05/05/2020 21:18:26 - INFO - __main__ -     step:27 loss 0.148\n",
      "05/05/2020 21:18:27 - INFO - __main__ -     step:28 loss 0.184\n",
      "05/05/2020 21:18:29 - INFO - __main__ -     step:29 loss 0.207\n",
      "05/05/2020 21:18:30 - INFO - __main__ -     step:30 loss 0.237\n",
      "05/05/2020 21:18:31 - INFO - __main__ -     step:31 loss 0.187\n",
      "05/05/2020 21:18:33 - INFO - __main__ -     step:32 loss 0.264\n",
      "05/05/2020 21:18:34 - INFO - __main__ -     step:33 loss 0.190\n",
      "05/05/2020 21:18:35 - INFO - __main__ -     step:34 loss 0.157\n",
      "05/05/2020 21:18:36 - INFO - __main__ -     step:35 loss 0.127\n",
      "05/05/2020 21:18:38 - INFO - __main__ -     step:36 loss 0.201\n",
      "05/05/2020 21:18:39 - INFO - __main__ -     step:37 loss 0.211\n",
      "05/05/2020 21:18:40 - INFO - __main__ -     step:38 loss 0.154\n",
      "05/05/2020 21:18:41 - INFO - __main__ -     step:39 loss 0.178\n",
      "05/05/2020 21:18:43 - INFO - __main__ -     step:40 loss 0.187\n",
      "05/05/2020 21:18:44 - INFO - __main__ -     step:41 loss 0.162\n",
      "05/05/2020 21:18:45 - INFO - __main__ -     step:42 loss 0.153\n",
      "05/05/2020 21:18:46 - INFO - __main__ -     step:43 loss 0.137\n",
      "05/05/2020 21:18:48 - INFO - __main__ -     step:44 loss 0.167\n",
      "05/05/2020 21:18:49 - INFO - __main__ -     step:45 loss 0.157\n",
      "05/05/2020 21:18:50 - INFO - __main__ -     step:46 loss 0.123\n",
      "05/05/2020 21:18:51 - INFO - __main__ -     step:47 loss 0.109\n",
      "05/05/2020 21:18:53 - INFO - __main__ -     step:48 loss 0.187\n",
      "05/05/2020 21:18:54 - INFO - __main__ -     step:49 loss 0.152\n",
      "05/05/2020 21:18:55 - INFO - __main__ -     step:50 loss 0.212\n",
      "05/05/2020 21:18:57 - INFO - __main__ -     step:51 loss 0.194\n",
      "05/05/2020 21:18:58 - INFO - __main__ -     step:52 loss 0.163\n",
      "05/05/2020 21:18:59 - INFO - __main__ -     step:53 loss 0.177\n",
      "05/05/2020 21:19:01 - INFO - __main__ -     step:54 loss 0.108\n",
      "05/05/2020 21:19:02 - INFO - __main__ -     step:55 loss 0.205\n",
      "05/05/2020 21:19:03 - INFO - __main__ -     step:56 loss 0.197\n",
      "05/05/2020 21:19:04 - INFO - __main__ -     step:57 loss 0.186\n",
      "05/05/2020 21:19:06 - INFO - __main__ -     step:58 loss 0.181\n",
      "05/05/2020 21:19:07 - INFO - __main__ -     step:59 loss 0.152\n",
      "05/05/2020 21:19:08 - INFO - __main__ -     step:60 loss 0.194\n",
      "05/05/2020 21:19:09 - INFO - __main__ -     step:61 loss 0.135\n",
      "05/05/2020 21:19:11 - INFO - __main__ -     step:62 loss 0.153\n",
      "05/05/2020 21:19:12 - INFO - __main__ -     step:63 loss 0.140\n",
      "05/05/2020 21:19:13 - INFO - __main__ -     step:64 loss 0.191\n",
      "05/05/2020 21:19:15 - INFO - __main__ -     step:65 loss 0.183\n",
      "05/05/2020 21:19:16 - INFO - __main__ -     step:66 loss 0.118\n",
      "05/05/2020 21:19:17 - INFO - __main__ -     step:67 loss 0.130\n",
      "05/05/2020 21:19:18 - INFO - __main__ -     step:68 loss 0.155\n",
      "05/05/2020 21:19:20 - INFO - __main__ -     step:69 loss 0.119\n",
      "05/05/2020 21:19:21 - INFO - __main__ -     step:70 loss 0.118\n",
      "05/05/2020 21:19:22 - INFO - __main__ -     step:71 loss 0.263\n",
      "05/05/2020 21:19:24 - INFO - __main__ -     step:72 loss 0.127\n",
      "05/05/2020 21:19:25 - INFO - __main__ -     step:73 loss 0.201\n",
      "05/05/2020 21:19:26 - INFO - __main__ -     step:74 loss 0.170\n",
      "05/05/2020 21:19:27 - INFO - __main__ -     step:75 loss 0.220\n",
      "05/05/2020 21:19:29 - INFO - __main__ -     step:76 loss 0.206\n",
      "05/05/2020 21:19:30 - INFO - __main__ -     step:77 loss 0.126\n",
      "05/05/2020 21:19:31 - INFO - __main__ -     step:78 loss 0.167\n",
      "05/05/2020 21:19:33 - INFO - __main__ -     step:79 loss 0.161\n",
      "05/05/2020 21:19:34 - INFO - __main__ -     step:80 loss 0.159\n",
      "05/05/2020 21:19:35 - INFO - __main__ -     step:81 loss 0.232\n",
      "05/05/2020 21:19:36 - INFO - __main__ -     step:82 loss 0.127\n",
      "05/05/2020 21:19:38 - INFO - __main__ -     step:83 loss 0.157\n",
      "05/05/2020 21:19:39 - INFO - __main__ -     step:84 loss 0.176\n",
      "05/05/2020 21:19:40 - INFO - __main__ -     step:85 loss 0.172\n",
      "05/05/2020 21:19:42 - INFO - __main__ -     step:86 loss 0.182\n",
      "05/05/2020 21:19:43 - INFO - __main__ -     step:87 loss 0.246\n",
      "05/05/2020 21:19:44 - INFO - __main__ -     step:88 loss 0.178\n",
      "05/05/2020 21:19:45 - INFO - __main__ -     step:89 loss 0.173\n",
      "05/05/2020 21:19:46 - INFO - __main__ -     step:90 loss 0.196\n",
      "05/05/2020 21:19:48 - INFO - __main__ -     step:91 loss 0.159\n",
      "05/05/2020 21:19:49 - INFO - __main__ -     step:92 loss 0.128\n",
      "05/05/2020 21:19:50 - INFO - __main__ -     step:93 loss 0.140\n",
      "05/05/2020 21:19:52 - INFO - __main__ -     step:94 loss 0.225\n",
      "05/05/2020 21:19:53 - INFO - __main__ -     step:95 loss 0.178\n",
      "05/05/2020 21:19:54 - INFO - __main__ -     step:96 loss 0.175\n",
      "05/05/2020 21:19:55 - INFO - __main__ -     step:97 loss 0.136\n",
      "05/05/2020 21:19:57 - INFO - __main__ -     step:98 loss 0.170\n",
      "05/05/2020 21:19:58 - INFO - __main__ -     step:99 loss 0.137\n",
      "05/05/2020 21:19:59 - INFO - __main__ -     step:100 loss 0.109\n",
      "05/05/2020 21:20:01 - INFO - __main__ -     step:101 loss 0.168\n",
      "05/05/2020 21:20:02 - INFO - __main__ -     step:102 loss 0.213\n",
      "05/05/2020 21:20:03 - INFO - __main__ -     step:103 loss 0.146\n",
      "05/05/2020 21:20:04 - INFO - __main__ -     step:104 loss 0.156\n",
      "05/05/2020 21:20:06 - INFO - __main__ -     step:105 loss 0.173\n",
      "05/05/2020 21:20:07 - INFO - __main__ -     step:106 loss 0.173\n",
      "05/05/2020 21:20:08 - INFO - __main__ -     step:107 loss 0.164\n",
      "05/05/2020 21:20:09 - INFO - __main__ -     step:108 loss 0.104\n",
      "05/05/2020 21:20:11 - INFO - __main__ -     step:109 loss 0.109\n",
      "05/05/2020 21:20:12 - INFO - __main__ -     step:110 loss 0.168\n",
      "05/05/2020 21:20:13 - INFO - __main__ -     step:111 loss 0.166\n",
      "05/05/2020 21:20:15 - INFO - __main__ -     step:112 loss 0.172\n",
      "05/05/2020 21:20:16 - INFO - __main__ -     step:113 loss 0.151\n",
      "05/05/2020 21:20:17 - INFO - __main__ -     step:114 loss 0.115\n",
      "05/05/2020 21:20:18 - INFO - __main__ -     step:115 loss 0.149\n",
      "05/05/2020 21:20:20 - INFO - __main__ -     step:116 loss 0.160\n",
      "05/05/2020 21:20:21 - INFO - __main__ -     step:117 loss 0.124\n",
      "05/05/2020 21:20:22 - INFO - __main__ -     step:118 loss 0.189\n",
      "05/05/2020 21:20:24 - INFO - __main__ -     step:119 loss 0.103\n",
      "05/05/2020 21:20:25 - INFO - __main__ -     step:120 loss 0.203\n",
      "05/05/2020 21:20:26 - INFO - __main__ -     step:121 loss 0.135\n",
      "05/05/2020 21:20:27 - INFO - __main__ -     step:122 loss 0.195\n",
      "05/05/2020 21:20:29 - INFO - __main__ -     step:123 loss 0.173\n",
      "05/05/2020 21:20:30 - INFO - __main__ -     step:124 loss 0.139\n",
      "05/05/2020 21:20:31 - INFO - __main__ -     step:125 loss 0.205\n",
      "05/05/2020 21:20:32 - INFO - __main__ -     step:126 loss 0.159\n",
      "05/05/2020 21:20:34 - INFO - __main__ -     step:127 loss 0.142\n",
      "05/05/2020 21:20:35 - INFO - __main__ -     step:128 loss 0.194\n",
      "05/05/2020 21:20:36 - INFO - __main__ -     step:129 loss 0.131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:20:37 - INFO - __main__ -     step:130 loss 0.182\n",
      "05/05/2020 21:20:39 - INFO - __main__ -     step:131 loss 0.146\n",
      "05/05/2020 21:20:40 - INFO - __main__ -     step:132 loss 0.233\n",
      "05/05/2020 21:20:41 - INFO - __main__ -     step:133 loss 0.150\n",
      "05/05/2020 21:20:43 - INFO - __main__ -     step:134 loss 0.099\n",
      "05/05/2020 21:20:44 - INFO - __main__ -     step:135 loss 0.173\n",
      "05/05/2020 21:20:45 - INFO - __main__ -     step:136 loss 0.205\n",
      "05/05/2020 21:20:47 - INFO - __main__ -     step:137 loss 0.135\n",
      "05/05/2020 21:20:48 - INFO - __main__ -     step:138 loss 0.181\n",
      "05/05/2020 21:20:49 - INFO - __main__ -     step:139 loss 0.260\n",
      "05/05/2020 21:20:50 - INFO - __main__ -     step:140 loss 0.133\n",
      "05/05/2020 21:20:52 - INFO - __main__ -     step:141 loss 0.186\n",
      "05/05/2020 21:20:53 - INFO - __main__ -     step:142 loss 0.102\n",
      "05/05/2020 21:20:54 - INFO - __main__ -     step:143 loss 0.178\n",
      "05/05/2020 21:20:55 - INFO - __main__ -     step:144 loss 0.180\n",
      "05/05/2020 21:20:57 - INFO - __main__ -     step:145 loss 0.172\n",
      "05/05/2020 21:20:58 - INFO - __main__ -     step:146 loss 0.134\n",
      "05/05/2020 21:20:59 - INFO - __main__ -     step:147 loss 0.121\n",
      "05/05/2020 21:21:01 - INFO - __main__ -     step:148 loss 0.179\n",
      "05/05/2020 21:21:02 - INFO - __main__ -     step:149 loss 0.195\n",
      "05/05/2020 21:21:03 - INFO - __main__ -     step:150 loss 0.117\n",
      "05/05/2020 21:21:04 - INFO - __main__ -     step:151 loss 0.171\n",
      "05/05/2020 21:21:06 - INFO - __main__ -     step:152 loss 0.125\n",
      "05/05/2020 21:21:07 - INFO - __main__ -     step:153 loss 0.119\n",
      "05/05/2020 21:21:08 - INFO - __main__ -     step:154 loss 0.241\n",
      "05/05/2020 21:21:10 - INFO - __main__ -     step:155 loss 0.118\n",
      "05/05/2020 21:21:11 - INFO - __main__ -     step:156 loss 0.132\n",
      "05/05/2020 21:21:12 - INFO - __main__ -     step:157 loss 0.103\n",
      "05/05/2020 21:21:13 - INFO - __main__ -     step:158 loss 0.197\n",
      "05/05/2020 21:21:15 - INFO - __main__ -     step:159 loss 0.139\n",
      "05/05/2020 21:21:16 - INFO - __main__ -     step:160 loss 0.116\n",
      "05/05/2020 21:21:17 - INFO - __main__ -     step:161 loss 0.131\n",
      "05/05/2020 21:21:18 - INFO - __main__ -     step:162 loss 0.133\n",
      "05/05/2020 21:21:20 - INFO - __main__ -     step:163 loss 0.160\n",
      "05/05/2020 21:21:21 - INFO - __main__ -     step:164 loss 0.214\n",
      "05/05/2020 21:21:22 - INFO - __main__ -     step:165 loss 0.143\n",
      "05/05/2020 21:21:23 - INFO - __main__ -     step:166 loss 0.235\n",
      "05/05/2020 21:21:25 - INFO - __main__ -     step:167 loss 0.146\n",
      "05/05/2020 21:21:26 - INFO - __main__ -     step:168 loss 0.108\n",
      "05/05/2020 21:21:27 - INFO - __main__ -     step:169 loss 0.150\n",
      "05/05/2020 21:21:28 - INFO - __main__ -     step:170 loss 0.143\n",
      "05/05/2020 21:21:30 - INFO - __main__ -     step:171 loss 0.208\n",
      "05/05/2020 21:21:31 - INFO - __main__ -     step:172 loss 0.175\n",
      "05/05/2020 21:21:32 - INFO - __main__ -     step:173 loss 0.128\n",
      "05/05/2020 21:21:33 - INFO - __main__ -     step:174 loss 0.124\n",
      "05/05/2020 21:21:35 - INFO - __main__ -     step:175 loss 0.140\n",
      "05/05/2020 21:21:36 - INFO - __main__ -     step:176 loss 0.164\n",
      "05/05/2020 21:21:37 - INFO - __main__ -     step:177 loss 0.166\n",
      "05/05/2020 21:21:39 - INFO - __main__ -     step:178 loss 0.132\n",
      "05/05/2020 21:21:40 - INFO - __main__ -     step:179 loss 0.147\n",
      "05/05/2020 21:21:41 - INFO - __main__ -     step:180 loss 0.156\n",
      "05/05/2020 21:21:43 - INFO - __main__ -     step:181 loss 0.171\n",
      "05/05/2020 21:21:44 - INFO - __main__ -     step:182 loss 0.136\n",
      "05/05/2020 21:21:45 - INFO - __main__ -     step:183 loss 0.084\n",
      "05/05/2020 21:21:46 - INFO - __main__ -     step:184 loss 0.217\n",
      "05/05/2020 21:21:48 - INFO - __main__ -     step:185 loss 0.259\n",
      "05/05/2020 21:21:49 - INFO - __main__ -     step:186 loss 0.166\n",
      "05/05/2020 21:21:50 - INFO - __main__ -     step:187 loss 0.158\n",
      "05/05/2020 21:21:51 - INFO - __main__ -     step:188 loss 0.098\n",
      "05/05/2020 21:21:53 - INFO - __main__ -     step:189 loss 0.134\n",
      "05/05/2020 21:21:54 - INFO - __main__ -     step:190 loss 0.182\n",
      "05/05/2020 21:21:55 - INFO - __main__ -     step:191 loss 0.101\n",
      "05/05/2020 21:21:57 - INFO - __main__ -     step:192 loss 0.095\n",
      "05/05/2020 21:21:58 - INFO - __main__ -     step:193 loss 0.109\n",
      "05/05/2020 21:21:59 - INFO - __main__ -     step:194 loss 0.128\n",
      "05/05/2020 21:22:01 - INFO - __main__ -     step:195 loss 0.126\n",
      "05/05/2020 21:22:02 - INFO - __main__ -     step:196 loss 0.147\n",
      "05/05/2020 21:22:03 - INFO - __main__ -     step:197 loss 0.128\n",
      "05/05/2020 21:22:04 - INFO - __main__ -     step:198 loss 0.134\n",
      "05/05/2020 21:22:06 - INFO - __main__ -     step:199 loss 0.211\n",
      "05/05/2020 21:22:07 - INFO - __main__ -     step:200 loss 0.173\n",
      "05/05/2020 21:22:08 - INFO - __main__ -     step:201 loss 0.129\n",
      "05/05/2020 21:22:10 - INFO - __main__ -     step:202 loss 0.132\n",
      "05/05/2020 21:22:11 - INFO - __main__ -     step:203 loss 0.207\n",
      "05/05/2020 21:22:12 - INFO - __main__ -     step:204 loss 0.150\n",
      "05/05/2020 21:22:13 - INFO - __main__ -     step:205 loss 0.087\n",
      "05/05/2020 21:22:15 - INFO - __main__ -     step:206 loss 0.104\n",
      "05/05/2020 21:22:16 - INFO - __main__ -     step:207 loss 0.187\n",
      "05/05/2020 21:22:17 - INFO - __main__ -     step:208 loss 0.152\n",
      "05/05/2020 21:22:18 - INFO - __main__ -     step:209 loss 0.269\n",
      "05/05/2020 21:22:20 - INFO - __main__ -     step:210 loss 0.125\n",
      "05/05/2020 21:22:21 - INFO - __main__ -     step:211 loss 0.186\n",
      "05/05/2020 21:22:22 - INFO - __main__ -     step:212 loss 0.146\n",
      "05/05/2020 21:22:24 - INFO - __main__ -     step:213 loss 0.237\n",
      "05/05/2020 21:22:25 - INFO - __main__ -     step:214 loss 0.119\n",
      "05/05/2020 21:22:26 - INFO - __main__ -     step:215 loss 0.140\n",
      "05/05/2020 21:22:27 - INFO - __main__ -     step:216 loss 0.212\n",
      "05/05/2020 21:22:29 - INFO - __main__ -     step:217 loss 0.120\n",
      "05/05/2020 21:22:30 - INFO - __main__ -     step:218 loss 0.142\n",
      "05/05/2020 21:22:31 - INFO - __main__ -     step:219 loss 0.116\n",
      "05/05/2020 21:22:32 - INFO - __main__ -     step:220 loss 0.113\n",
      "05/05/2020 21:22:34 - INFO - __main__ -     step:221 loss 0.171\n",
      "05/05/2020 21:22:35 - INFO - __main__ -     step:222 loss 0.138\n",
      "05/05/2020 21:22:36 - INFO - __main__ -     step:223 loss 0.195\n",
      "05/05/2020 21:22:38 - INFO - __main__ -     step:224 loss 0.190\n",
      "05/05/2020 21:22:39 - INFO - __main__ -     step:225 loss 0.146\n",
      "05/05/2020 21:22:40 - INFO - __main__ -     step:226 loss 0.190\n",
      "05/05/2020 21:22:41 - INFO - __main__ -     step:227 loss 0.108\n",
      "05/05/2020 21:22:43 - INFO - __main__ -     step:228 loss 0.169\n",
      "05/05/2020 21:22:44 - INFO - __main__ -     step:229 loss 0.118\n",
      "05/05/2020 21:22:45 - INFO - __main__ -     step:230 loss 0.141\n",
      "05/05/2020 21:22:47 - INFO - __main__ -     step:231 loss 0.163\n",
      "05/05/2020 21:22:48 - INFO - __main__ -     step:232 loss 0.142\n",
      "05/05/2020 21:22:49 - INFO - __main__ -     step:233 loss 0.101\n",
      "05/05/2020 21:22:50 - INFO - __main__ -     step:234 loss 0.100\n",
      "05/05/2020 21:22:52 - INFO - __main__ -     step:235 loss 0.169\n",
      "05/05/2020 21:22:53 - INFO - __main__ -     step:236 loss 0.080\n",
      "05/05/2020 21:22:54 - INFO - __main__ -     step:237 loss 0.169\n",
      "05/05/2020 21:22:56 - INFO - __main__ -     step:238 loss 0.154\n",
      "05/05/2020 21:22:57 - INFO - __main__ -     step:239 loss 0.158\n",
      "05/05/2020 21:22:58 - INFO - __main__ -     step:240 loss 0.142\n",
      "05/05/2020 21:22:59 - INFO - __main__ -     step:241 loss 0.168\n",
      "05/05/2020 21:23:01 - INFO - __main__ -     step:242 loss 0.124\n",
      "05/05/2020 21:23:02 - INFO - __main__ -     step:243 loss 0.149\n",
      "05/05/2020 21:23:03 - INFO - __main__ -     step:244 loss 0.126\n",
      "05/05/2020 21:23:05 - INFO - __main__ -     step:245 loss 0.110\n",
      "05/05/2020 21:23:06 - INFO - __main__ -     step:246 loss 0.132\n",
      "05/05/2020 21:23:07 - INFO - __main__ -     step:247 loss 0.151\n",
      "05/05/2020 21:23:08 - INFO - __main__ -     step:248 loss 0.135\n",
      "05/05/2020 21:23:10 - INFO - __main__ -     step:249 loss 0.120\n",
      "05/05/2020 21:23:11 - INFO - __main__ -     step:250 loss 0.214\n",
      "05/05/2020 21:23:12 - INFO - __main__ -     step:251 loss 0.151\n",
      "05/05/2020 21:23:14 - INFO - __main__ -     step:252 loss 0.198\n",
      "05/05/2020 21:23:15 - INFO - __main__ -     step:253 loss 0.116\n",
      "05/05/2020 21:23:16 - INFO - __main__ -     step:254 loss 0.184\n",
      "05/05/2020 21:23:17 - INFO - __main__ -     step:255 loss 0.125\n",
      "05/05/2020 21:23:19 - INFO - __main__ -     step:256 loss 0.120\n",
      "05/05/2020 21:23:20 - INFO - __main__ -     step:257 loss 0.195\n",
      "05/05/2020 21:23:21 - INFO - __main__ -     step:258 loss 0.230\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:23:22 - INFO - __main__ -     step:259 loss 0.173\n",
      "05/05/2020 21:23:24 - INFO - __main__ -     step:260 loss 0.128\n",
      "05/05/2020 21:23:25 - INFO - __main__ -     step:261 loss 0.136\n",
      "05/05/2020 21:23:26 - INFO - __main__ -     step:262 loss 0.109\n",
      "05/05/2020 21:23:28 - INFO - __main__ -     step:263 loss 0.107\n",
      "05/05/2020 21:23:29 - INFO - __main__ -     step:264 loss 0.150\n",
      "05/05/2020 21:23:30 - INFO - __main__ -     step:265 loss 0.160\n",
      "05/05/2020 21:23:31 - INFO - __main__ -     step:266 loss 0.145\n",
      "05/05/2020 21:23:33 - INFO - __main__ -     step:267 loss 0.108\n",
      "05/05/2020 21:23:34 - INFO - __main__ -     step:268 loss 0.120\n",
      "05/05/2020 21:23:35 - INFO - __main__ -     step:269 loss 0.096\n",
      "05/05/2020 21:23:36 - INFO - __main__ -     step:270 loss 0.126\n",
      "05/05/2020 21:23:38 - INFO - __main__ -     step:271 loss 0.196\n",
      "05/05/2020 21:23:39 - INFO - __main__ -     step:272 loss 0.142\n",
      "05/05/2020 21:23:40 - INFO - __main__ -     step:273 loss 0.142\n",
      "05/05/2020 21:23:42 - INFO - __main__ -     step:274 loss 0.212\n",
      "05/05/2020 21:23:43 - INFO - __main__ -     step:275 loss 0.129\n",
      "05/05/2020 21:23:44 - INFO - __main__ -     step:276 loss 0.170\n",
      "05/05/2020 21:23:45 - INFO - __main__ -     step:277 loss 0.144\n",
      "05/05/2020 21:23:47 - INFO - __main__ -     step:278 loss 0.112\n",
      "05/05/2020 21:23:48 - INFO - __main__ -     step:279 loss 0.148\n",
      "05/05/2020 21:23:49 - INFO - __main__ -     step:280 loss 0.150\n",
      "05/05/2020 21:23:50 - INFO - __main__ -     step:281 loss 0.140\n",
      "05/05/2020 21:23:52 - INFO - __main__ -     step:282 loss 0.227\n",
      "05/05/2020 21:23:53 - INFO - __main__ -     step:283 loss 0.164\n",
      "05/05/2020 21:23:54 - INFO - __main__ -     step:284 loss 0.127\n",
      "05/05/2020 21:23:55 - INFO - __main__ -     step:285 loss 0.220\n",
      "05/05/2020 21:23:57 - INFO - __main__ -     step:286 loss 0.133\n",
      "05/05/2020 21:23:58 - INFO - __main__ -     step:287 loss 0.118\n",
      "05/05/2020 21:23:59 - INFO - __main__ -     step:288 loss 0.115\n",
      "05/05/2020 21:24:00 - INFO - __main__ -     step:289 loss 0.138\n",
      "05/05/2020 21:24:02 - INFO - __main__ -     step:290 loss 0.093\n",
      "05/05/2020 21:24:03 - INFO - __main__ -     step:291 loss 0.115\n",
      "05/05/2020 21:24:04 - INFO - __main__ -     step:292 loss 0.118\n",
      "05/05/2020 21:24:05 - INFO - __main__ -     step:293 loss 0.117\n",
      "05/05/2020 21:24:07 - INFO - __main__ -     step:294 loss 0.149\n",
      "05/05/2020 21:24:08 - INFO - __main__ -     step:295 loss 0.145\n",
      "05/05/2020 21:24:09 - INFO - __main__ -     step:296 loss 0.190\n",
      "05/05/2020 21:24:11 - INFO - __main__ -     step:297 loss 0.102\n",
      "05/05/2020 21:24:12 - INFO - __main__ -     step:298 loss 0.130\n",
      "05/05/2020 21:24:13 - INFO - __main__ -     step:299 loss 0.127\n",
      "05/05/2020 21:24:15 - INFO - __main__ -     step:300 loss 0.165\n",
      "05/05/2020 21:24:16 - INFO - __main__ -     step:301 loss 0.185\n",
      "05/05/2020 21:24:17 - INFO - __main__ -     step:302 loss 0.108\n",
      "05/05/2020 21:24:18 - INFO - __main__ -     step:303 loss 0.173\n",
      "05/05/2020 21:24:20 - INFO - __main__ -     step:304 loss 0.111\n",
      "05/05/2020 21:24:21 - INFO - __main__ -     step:305 loss 0.150\n",
      "05/05/2020 21:24:22 - INFO - __main__ -     step:306 loss 0.075\n",
      "05/05/2020 21:24:23 - INFO - __main__ -     step:307 loss 0.161\n",
      "05/05/2020 21:24:25 - INFO - __main__ -     step:308 loss 0.175\n",
      "05/05/2020 21:24:26 - INFO - __main__ -     step:309 loss 0.163\n",
      "05/05/2020 21:24:27 - INFO - __main__ -     step:310 loss 0.113\n",
      "05/05/2020 21:24:29 - INFO - __main__ -     step:311 loss 0.138\n",
      "05/05/2020 21:24:29 - INFO - __main__ -     step:312 loss 0.128\n",
      "05/05/2020 21:24:29 - INFO - __main__ -   ***** Running evaluation  *****\n",
      "05/05/2020 21:24:29 - INFO - __main__ -     Num examples = 1000\n",
      "05/05/2020 21:24:29 - INFO - __main__ -     Batch size = 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33e2b998acbd45908cb6ce166d67bd5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=32, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:24:40 - INFO - __main__ -   ***** Eval results  *****\n",
      "05/05/2020 21:24:40 - INFO - __main__ -     acc_class = [0.966 0.991 0.999 0.983 0.935 0.985 0.985 0.997 0.989 0.907 0.918 0.987\n",
      " 0.984 0.897 0.989 0.998 0.945 0.961 0.92  1.    0.92  0.99  0.959 0.955\n",
      " 0.996 0.976 0.976 0.992 0.972 0.965 0.959 1.    0.999 1.    0.941 0.917\n",
      " 0.976 0.996 0.944 0.987 1.    0.985 0.935 0.986 1.    0.93  0.946 0.996\n",
      " 0.973 0.961 1.    0.959 0.989 0.854 0.99 ]\n",
      "05/05/2020 21:24:40 - INFO - __main__ -     acc_sample = 0.274\n",
      "05/05/2020 21:24:40 - INFO - __main__ -     precision_sample = 0.47338920840672005\n",
      "05/05/2020 21:24:40 - INFO - __main__ -     recall_sample = 0.9694072657743786\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:24:43 - INFO - __main__ -   Saving model checkpoint to ./output_test/checkpoint-626\n",
      "05/05/2020 21:24:52 - INFO - __main__ -   Saving optimizer and scheduler states to ./output_test/checkpoint-626\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "420.4650185108185\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7806a7eeb5ab4d9ea18fd19194a11cd4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Iteration', max=313, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:24:53 - INFO - __main__ -     step:0 loss 0.106\n",
      "05/05/2020 21:24:55 - INFO - __main__ -     step:1 loss 0.127\n",
      "05/05/2020 21:24:56 - INFO - __main__ -     step:2 loss 0.075\n",
      "05/05/2020 21:24:57 - INFO - __main__ -     step:3 loss 0.130\n",
      "05/05/2020 21:24:58 - INFO - __main__ -     step:4 loss 0.116\n",
      "05/05/2020 21:24:59 - INFO - __main__ -     step:5 loss 0.126\n",
      "05/05/2020 21:25:00 - INFO - __main__ -     step:6 loss 0.108\n",
      "05/05/2020 21:25:02 - INFO - __main__ -     step:7 loss 0.125\n",
      "05/05/2020 21:25:03 - INFO - __main__ -     step:8 loss 0.125\n",
      "05/05/2020 21:25:04 - INFO - __main__ -     step:9 loss 0.092\n",
      "05/05/2020 21:25:05 - INFO - __main__ -     step:10 loss 0.065\n",
      "05/05/2020 21:25:06 - INFO - __main__ -     step:11 loss 0.116\n",
      "05/05/2020 21:25:08 - INFO - __main__ -     step:12 loss 0.099\n",
      "05/05/2020 21:25:09 - INFO - __main__ -     step:13 loss 0.151\n",
      "05/05/2020 21:25:10 - INFO - __main__ -     step:14 loss 0.115\n",
      "05/05/2020 21:25:11 - INFO - __main__ -     step:15 loss 0.118\n",
      "05/05/2020 21:25:12 - INFO - __main__ -     step:16 loss 0.079\n",
      "05/05/2020 21:25:14 - INFO - __main__ -     step:17 loss 0.117\n",
      "05/05/2020 21:25:15 - INFO - __main__ -     step:18 loss 0.119\n",
      "05/05/2020 21:25:16 - INFO - __main__ -     step:19 loss 0.094\n",
      "05/05/2020 21:25:18 - INFO - __main__ -     step:20 loss 0.138\n",
      "05/05/2020 21:25:19 - INFO - __main__ -     step:21 loss 0.139\n",
      "05/05/2020 21:25:20 - INFO - __main__ -     step:22 loss 0.124\n",
      "05/05/2020 21:25:21 - INFO - __main__ -     step:23 loss 0.087\n",
      "05/05/2020 21:25:23 - INFO - __main__ -     step:24 loss 0.088\n",
      "05/05/2020 21:25:24 - INFO - __main__ -     step:25 loss 0.124\n",
      "05/05/2020 21:25:25 - INFO - __main__ -     step:26 loss 0.077\n",
      "05/05/2020 21:25:27 - INFO - __main__ -     step:27 loss 0.116\n",
      "05/05/2020 21:25:28 - INFO - __main__ -     step:28 loss 0.077\n",
      "05/05/2020 21:25:29 - INFO - __main__ -     step:29 loss 0.129\n",
      "05/05/2020 21:25:30 - INFO - __main__ -     step:30 loss 0.096\n",
      "05/05/2020 21:25:32 - INFO - __main__ -     step:31 loss 0.050\n",
      "05/05/2020 21:25:33 - INFO - __main__ -     step:32 loss 0.105\n",
      "05/05/2020 21:25:34 - INFO - __main__ -     step:33 loss 0.083\n",
      "05/05/2020 21:25:36 - INFO - __main__ -     step:34 loss 0.132\n",
      "05/05/2020 21:25:37 - INFO - __main__ -     step:35 loss 0.090\n",
      "05/05/2020 21:25:38 - INFO - __main__ -     step:36 loss 0.086\n",
      "05/05/2020 21:25:39 - INFO - __main__ -     step:37 loss 0.082\n",
      "05/05/2020 21:25:41 - INFO - __main__ -     step:38 loss 0.121\n",
      "05/05/2020 21:25:42 - INFO - __main__ -     step:39 loss 0.136\n",
      "05/05/2020 21:25:43 - INFO - __main__ -     step:40 loss 0.064\n",
      "05/05/2020 21:25:44 - INFO - __main__ -     step:41 loss 0.110\n",
      "05/05/2020 21:25:46 - INFO - __main__ -     step:42 loss 0.136\n",
      "05/05/2020 21:25:47 - INFO - __main__ -     step:43 loss 0.074\n",
      "05/05/2020 21:25:48 - INFO - __main__ -     step:44 loss 0.106\n",
      "05/05/2020 21:25:49 - INFO - __main__ -     step:45 loss 0.107\n",
      "05/05/2020 21:25:51 - INFO - __main__ -     step:46 loss 0.220\n",
      "05/05/2020 21:25:52 - INFO - __main__ -     step:47 loss 0.105\n",
      "05/05/2020 21:25:53 - INFO - __main__ -     step:48 loss 0.057\n",
      "05/05/2020 21:25:55 - INFO - __main__ -     step:49 loss 0.066\n",
      "05/05/2020 21:25:56 - INFO - __main__ -     step:50 loss 0.094\n",
      "05/05/2020 21:25:57 - INFO - __main__ -     step:51 loss 0.086\n",
      "05/05/2020 21:25:59 - INFO - __main__ -     step:52 loss 0.140\n",
      "05/05/2020 21:26:00 - INFO - __main__ -     step:53 loss 0.071\n",
      "05/05/2020 21:26:01 - INFO - __main__ -     step:54 loss 0.070\n",
      "05/05/2020 21:26:03 - INFO - __main__ -     step:55 loss 0.075\n",
      "05/05/2020 21:26:04 - INFO - __main__ -     step:56 loss 0.064\n",
      "05/05/2020 21:26:05 - INFO - __main__ -     step:57 loss 0.087\n",
      "05/05/2020 21:26:06 - INFO - __main__ -     step:58 loss 0.109\n",
      "05/05/2020 21:26:08 - INFO - __main__ -     step:59 loss 0.129\n",
      "05/05/2020 21:26:09 - INFO - __main__ -     step:60 loss 0.109\n",
      "05/05/2020 21:26:10 - INFO - __main__ -     step:61 loss 0.144\n",
      "05/05/2020 21:26:12 - INFO - __main__ -     step:62 loss 0.110\n",
      "05/05/2020 21:26:13 - INFO - __main__ -     step:63 loss 0.116\n",
      "05/05/2020 21:26:14 - INFO - __main__ -     step:64 loss 0.096\n",
      "05/05/2020 21:26:15 - INFO - __main__ -     step:65 loss 0.096\n",
      "05/05/2020 21:26:17 - INFO - __main__ -     step:66 loss 0.091\n",
      "05/05/2020 21:26:18 - INFO - __main__ -     step:67 loss 0.106\n",
      "05/05/2020 21:26:19 - INFO - __main__ -     step:68 loss 0.085\n",
      "05/05/2020 21:26:20 - INFO - __main__ -     step:69 loss 0.118\n",
      "05/05/2020 21:26:22 - INFO - __main__ -     step:70 loss 0.118\n",
      "05/05/2020 21:26:23 - INFO - __main__ -     step:71 loss 0.064\n",
      "05/05/2020 21:26:24 - INFO - __main__ -     step:72 loss 0.066\n",
      "05/05/2020 21:26:26 - INFO - __main__ -     step:73 loss 0.083\n",
      "05/05/2020 21:26:27 - INFO - __main__ -     step:74 loss 0.164\n",
      "05/05/2020 21:26:28 - INFO - __main__ -     step:75 loss 0.105\n",
      "05/05/2020 21:26:30 - INFO - __main__ -     step:76 loss 0.109\n",
      "05/05/2020 21:26:31 - INFO - __main__ -     step:77 loss 0.102\n",
      "05/05/2020 21:26:32 - INFO - __main__ -     step:78 loss 0.141\n",
      "05/05/2020 21:26:33 - INFO - __main__ -     step:79 loss 0.141\n",
      "05/05/2020 21:26:35 - INFO - __main__ -     step:80 loss 0.078\n",
      "05/05/2020 21:26:36 - INFO - __main__ -     step:81 loss 0.078\n",
      "05/05/2020 21:26:37 - INFO - __main__ -     step:82 loss 0.121\n",
      "05/05/2020 21:26:39 - INFO - __main__ -     step:83 loss 0.094\n",
      "05/05/2020 21:26:40 - INFO - __main__ -     step:84 loss 0.102\n",
      "05/05/2020 21:26:41 - INFO - __main__ -     step:85 loss 0.088\n",
      "05/05/2020 21:26:43 - INFO - __main__ -     step:86 loss 0.126\n",
      "05/05/2020 21:26:44 - INFO - __main__ -     step:87 loss 0.095\n",
      "05/05/2020 21:26:45 - INFO - __main__ -     step:88 loss 0.184\n",
      "05/05/2020 21:26:46 - INFO - __main__ -     step:89 loss 0.133\n",
      "05/05/2020 21:26:48 - INFO - __main__ -     step:90 loss 0.122\n",
      "05/05/2020 21:26:49 - INFO - __main__ -     step:91 loss 0.139\n",
      "05/05/2020 21:26:50 - INFO - __main__ -     step:92 loss 0.082\n",
      "05/05/2020 21:26:52 - INFO - __main__ -     step:93 loss 0.079\n",
      "05/05/2020 21:26:53 - INFO - __main__ -     step:94 loss 0.090\n",
      "05/05/2020 21:26:54 - INFO - __main__ -     step:95 loss 0.116\n",
      "05/05/2020 21:26:55 - INFO - __main__ -     step:96 loss 0.123\n",
      "05/05/2020 21:26:57 - INFO - __main__ -     step:97 loss 0.114\n",
      "05/05/2020 21:26:58 - INFO - __main__ -     step:98 loss 0.078\n",
      "05/05/2020 21:26:59 - INFO - __main__ -     step:99 loss 0.098\n",
      "05/05/2020 21:27:01 - INFO - __main__ -     step:100 loss 0.087\n",
      "05/05/2020 21:27:02 - INFO - __main__ -     step:101 loss 0.161\n",
      "05/05/2020 21:27:03 - INFO - __main__ -     step:102 loss 0.083\n",
      "05/05/2020 21:27:04 - INFO - __main__ -     step:103 loss 0.095\n",
      "05/05/2020 21:27:06 - INFO - __main__ -     step:104 loss 0.155\n",
      "05/05/2020 21:27:07 - INFO - __main__ -     step:105 loss 0.077\n",
      "05/05/2020 21:27:08 - INFO - __main__ -     step:106 loss 0.075\n",
      "05/05/2020 21:27:09 - INFO - __main__ -     step:107 loss 0.067\n",
      "05/05/2020 21:27:11 - INFO - __main__ -     step:108 loss 0.058\n",
      "05/05/2020 21:27:12 - INFO - __main__ -     step:109 loss 0.086\n",
      "05/05/2020 21:27:13 - INFO - __main__ -     step:110 loss 0.133\n",
      "05/05/2020 21:27:15 - INFO - __main__ -     step:111 loss 0.143\n",
      "05/05/2020 21:27:16 - INFO - __main__ -     step:112 loss 0.100\n",
      "05/05/2020 21:27:17 - INFO - __main__ -     step:113 loss 0.108\n",
      "05/05/2020 21:27:18 - INFO - __main__ -     step:114 loss 0.065\n",
      "05/05/2020 21:27:20 - INFO - __main__ -     step:115 loss 0.112\n",
      "05/05/2020 21:27:21 - INFO - __main__ -     step:116 loss 0.107\n",
      "05/05/2020 21:27:22 - INFO - __main__ -     step:117 loss 0.089\n",
      "05/05/2020 21:27:23 - INFO - __main__ -     step:118 loss 0.113\n",
      "05/05/2020 21:27:25 - INFO - __main__ -     step:119 loss 0.131\n",
      "05/05/2020 21:27:26 - INFO - __main__ -     step:120 loss 0.107\n",
      "05/05/2020 21:27:27 - INFO - __main__ -     step:121 loss 0.070\n",
      "05/05/2020 21:27:29 - INFO - __main__ -     step:122 loss 0.137\n",
      "05/05/2020 21:27:30 - INFO - __main__ -     step:123 loss 0.138\n",
      "05/05/2020 21:27:31 - INFO - __main__ -     step:124 loss 0.091\n",
      "05/05/2020 21:27:32 - INFO - __main__ -     step:125 loss 0.078\n",
      "05/05/2020 21:27:34 - INFO - __main__ -     step:126 loss 0.104\n",
      "05/05/2020 21:27:35 - INFO - __main__ -     step:127 loss 0.106\n",
      "05/05/2020 21:27:36 - INFO - __main__ -     step:128 loss 0.089\n",
      "05/05/2020 21:27:38 - INFO - __main__ -     step:129 loss 0.092\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:27:39 - INFO - __main__ -     step:130 loss 0.061\n",
      "05/05/2020 21:27:40 - INFO - __main__ -     step:131 loss 0.083\n",
      "05/05/2020 21:27:41 - INFO - __main__ -     step:132 loss 0.104\n",
      "05/05/2020 21:27:43 - INFO - __main__ -     step:133 loss 0.110\n",
      "05/05/2020 21:27:44 - INFO - __main__ -     step:134 loss 0.072\n",
      "05/05/2020 21:27:45 - INFO - __main__ -     step:135 loss 0.081\n",
      "05/05/2020 21:27:47 - INFO - __main__ -     step:136 loss 0.059\n",
      "05/05/2020 21:27:48 - INFO - __main__ -     step:137 loss 0.119\n",
      "05/05/2020 21:27:49 - INFO - __main__ -     step:138 loss 0.082\n",
      "05/05/2020 21:27:51 - INFO - __main__ -     step:139 loss 0.126\n",
      "05/05/2020 21:27:52 - INFO - __main__ -     step:140 loss 0.094\n",
      "05/05/2020 21:27:53 - INFO - __main__ -     step:141 loss 0.173\n",
      "05/05/2020 21:27:54 - INFO - __main__ -     step:142 loss 0.065\n",
      "05/05/2020 21:27:56 - INFO - __main__ -     step:143 loss 0.113\n",
      "05/05/2020 21:27:57 - INFO - __main__ -     step:144 loss 0.092\n",
      "05/05/2020 21:27:58 - INFO - __main__ -     step:145 loss 0.200\n",
      "05/05/2020 21:27:59 - INFO - __main__ -     step:146 loss 0.106\n",
      "05/05/2020 21:28:01 - INFO - __main__ -     step:147 loss 0.091\n",
      "05/05/2020 21:28:02 - INFO - __main__ -     step:148 loss 0.104\n",
      "05/05/2020 21:28:03 - INFO - __main__ -     step:149 loss 0.119\n",
      "05/05/2020 21:28:05 - INFO - __main__ -     step:150 loss 0.102\n",
      "05/05/2020 21:28:06 - INFO - __main__ -     step:151 loss 0.107\n",
      "05/05/2020 21:28:07 - INFO - __main__ -     step:152 loss 0.086\n",
      "05/05/2020 21:28:08 - INFO - __main__ -     step:153 loss 0.065\n",
      "05/05/2020 21:28:10 - INFO - __main__ -     step:154 loss 0.069\n",
      "05/05/2020 21:28:11 - INFO - __main__ -     step:155 loss 0.081\n",
      "05/05/2020 21:28:12 - INFO - __main__ -     step:156 loss 0.077\n",
      "05/05/2020 21:28:13 - INFO - __main__ -     step:157 loss 0.108\n",
      "05/05/2020 21:28:15 - INFO - __main__ -     step:158 loss 0.095\n",
      "05/05/2020 21:28:16 - INFO - __main__ -     step:159 loss 0.079\n",
      "05/05/2020 21:28:17 - INFO - __main__ -     step:160 loss 0.109\n",
      "05/05/2020 21:28:19 - INFO - __main__ -     step:161 loss 0.097\n",
      "05/05/2020 21:28:20 - INFO - __main__ -     step:162 loss 0.062\n",
      "05/05/2020 21:28:21 - INFO - __main__ -     step:163 loss 0.088\n",
      "05/05/2020 21:28:23 - INFO - __main__ -     step:164 loss 0.129\n",
      "05/05/2020 21:28:24 - INFO - __main__ -     step:165 loss 0.100\n",
      "05/05/2020 21:28:25 - INFO - __main__ -     step:166 loss 0.074\n",
      "05/05/2020 21:28:26 - INFO - __main__ -     step:167 loss 0.135\n",
      "05/05/2020 21:28:28 - INFO - __main__ -     step:168 loss 0.112\n",
      "05/05/2020 21:28:29 - INFO - __main__ -     step:169 loss 0.117\n",
      "05/05/2020 21:28:30 - INFO - __main__ -     step:170 loss 0.071\n",
      "05/05/2020 21:28:31 - INFO - __main__ -     step:171 loss 0.079\n",
      "05/05/2020 21:28:33 - INFO - __main__ -     step:172 loss 0.131\n",
      "05/05/2020 21:28:34 - INFO - __main__ -     step:173 loss 0.092\n",
      "05/05/2020 21:28:35 - INFO - __main__ -     step:174 loss 0.086\n",
      "05/05/2020 21:28:36 - INFO - __main__ -     step:175 loss 0.059\n",
      "05/05/2020 21:28:38 - INFO - __main__ -     step:176 loss 0.090\n",
      "05/05/2020 21:28:39 - INFO - __main__ -     step:177 loss 0.076\n",
      "05/05/2020 21:28:40 - INFO - __main__ -     step:178 loss 0.130\n",
      "05/05/2020 21:28:41 - INFO - __main__ -     step:179 loss 0.141\n",
      "05/05/2020 21:28:43 - INFO - __main__ -     step:180 loss 0.091\n",
      "05/05/2020 21:28:44 - INFO - __main__ -     step:181 loss 0.065\n",
      "05/05/2020 21:28:45 - INFO - __main__ -     step:182 loss 0.093\n",
      "05/05/2020 21:28:47 - INFO - __main__ -     step:183 loss 0.083\n",
      "05/05/2020 21:28:48 - INFO - __main__ -     step:184 loss 0.096\n",
      "05/05/2020 21:28:49 - INFO - __main__ -     step:185 loss 0.132\n",
      "05/05/2020 21:28:51 - INFO - __main__ -     step:186 loss 0.102\n",
      "05/05/2020 21:28:52 - INFO - __main__ -     step:187 loss 0.082\n",
      "05/05/2020 21:28:53 - INFO - __main__ -     step:188 loss 0.106\n",
      "05/05/2020 21:28:54 - INFO - __main__ -     step:189 loss 0.047\n",
      "05/05/2020 21:28:56 - INFO - __main__ -     step:190 loss 0.135\n",
      "05/05/2020 21:28:57 - INFO - __main__ -     step:191 loss 0.128\n",
      "05/05/2020 21:28:58 - INFO - __main__ -     step:192 loss 0.101\n",
      "05/05/2020 21:29:00 - INFO - __main__ -     step:193 loss 0.094\n",
      "05/05/2020 21:29:01 - INFO - __main__ -     step:194 loss 0.123\n",
      "05/05/2020 21:29:02 - INFO - __main__ -     step:195 loss 0.118\n",
      "05/05/2020 21:29:03 - INFO - __main__ -     step:196 loss 0.132\n",
      "05/05/2020 21:29:05 - INFO - __main__ -     step:197 loss 0.120\n",
      "05/05/2020 21:29:06 - INFO - __main__ -     step:198 loss 0.088\n",
      "05/05/2020 21:29:07 - INFO - __main__ -     step:199 loss 0.119\n",
      "05/05/2020 21:29:08 - INFO - __main__ -     step:200 loss 0.077\n",
      "05/05/2020 21:29:09 - INFO - __main__ -     step:201 loss 0.069\n",
      "05/05/2020 21:29:11 - INFO - __main__ -     step:202 loss 0.074\n",
      "05/05/2020 21:29:12 - INFO - __main__ -     step:203 loss 0.074\n",
      "05/05/2020 21:29:13 - INFO - __main__ -     step:204 loss 0.090\n",
      "05/05/2020 21:29:15 - INFO - __main__ -     step:205 loss 0.094\n",
      "05/05/2020 21:29:16 - INFO - __main__ -     step:206 loss 0.087\n",
      "05/05/2020 21:29:17 - INFO - __main__ -     step:207 loss 0.118\n",
      "05/05/2020 21:29:19 - INFO - __main__ -     step:208 loss 0.098\n",
      "05/05/2020 21:29:20 - INFO - __main__ -     step:209 loss 0.094\n",
      "05/05/2020 21:29:21 - INFO - __main__ -     step:210 loss 0.122\n",
      "05/05/2020 21:29:22 - INFO - __main__ -     step:211 loss 0.109\n",
      "05/05/2020 21:29:24 - INFO - __main__ -     step:212 loss 0.081\n",
      "05/05/2020 21:29:25 - INFO - __main__ -     step:213 loss 0.131\n",
      "05/05/2020 21:29:26 - INFO - __main__ -     step:214 loss 0.053\n",
      "05/05/2020 21:29:28 - INFO - __main__ -     step:215 loss 0.059\n",
      "05/05/2020 21:29:29 - INFO - __main__ -     step:216 loss 0.068\n",
      "05/05/2020 21:29:30 - INFO - __main__ -     step:217 loss 0.110\n",
      "05/05/2020 21:29:31 - INFO - __main__ -     step:218 loss 0.059\n",
      "05/05/2020 21:29:33 - INFO - __main__ -     step:219 loss 0.056\n",
      "05/05/2020 21:29:34 - INFO - __main__ -     step:220 loss 0.055\n",
      "05/05/2020 21:29:35 - INFO - __main__ -     step:221 loss 0.081\n",
      "05/05/2020 21:29:37 - INFO - __main__ -     step:222 loss 0.115\n",
      "05/05/2020 21:29:38 - INFO - __main__ -     step:223 loss 0.132\n",
      "05/05/2020 21:29:39 - INFO - __main__ -     step:224 loss 0.118\n",
      "05/05/2020 21:29:40 - INFO - __main__ -     step:225 loss 0.082\n",
      "05/05/2020 21:29:42 - INFO - __main__ -     step:226 loss 0.141\n",
      "05/05/2020 21:29:43 - INFO - __main__ -     step:227 loss 0.069\n",
      "05/05/2020 21:29:44 - INFO - __main__ -     step:228 loss 0.084\n",
      "05/05/2020 21:29:46 - INFO - __main__ -     step:229 loss 0.075\n",
      "05/05/2020 21:29:47 - INFO - __main__ -     step:230 loss 0.162\n",
      "05/05/2020 21:29:48 - INFO - __main__ -     step:231 loss 0.111\n",
      "05/05/2020 21:29:49 - INFO - __main__ -     step:232 loss 0.135\n",
      "05/05/2020 21:29:51 - INFO - __main__ -     step:233 loss 0.072\n",
      "05/05/2020 21:29:52 - INFO - __main__ -     step:234 loss 0.069\n",
      "05/05/2020 21:29:53 - INFO - __main__ -     step:235 loss 0.089\n",
      "05/05/2020 21:29:55 - INFO - __main__ -     step:236 loss 0.101\n",
      "05/05/2020 21:29:56 - INFO - __main__ -     step:237 loss 0.099\n",
      "05/05/2020 21:29:57 - INFO - __main__ -     step:238 loss 0.128\n",
      "05/05/2020 21:29:58 - INFO - __main__ -     step:239 loss 0.115\n",
      "05/05/2020 21:30:00 - INFO - __main__ -     step:240 loss 0.111\n",
      "05/05/2020 21:30:01 - INFO - __main__ -     step:241 loss 0.095\n",
      "05/05/2020 21:30:02 - INFO - __main__ -     step:242 loss 0.103\n",
      "05/05/2020 21:30:04 - INFO - __main__ -     step:243 loss 0.087\n",
      "05/05/2020 21:30:05 - INFO - __main__ -     step:244 loss 0.111\n",
      "05/05/2020 21:30:06 - INFO - __main__ -     step:245 loss 0.100\n",
      "05/05/2020 21:30:07 - INFO - __main__ -     step:246 loss 0.074\n",
      "05/05/2020 21:30:09 - INFO - __main__ -     step:247 loss 0.084\n",
      "05/05/2020 21:30:10 - INFO - __main__ -     step:248 loss 0.126\n",
      "05/05/2020 21:30:11 - INFO - __main__ -     step:249 loss 0.088\n",
      "05/05/2020 21:30:12 - INFO - __main__ -     step:250 loss 0.081\n",
      "05/05/2020 21:30:14 - INFO - __main__ -     step:251 loss 0.075\n",
      "05/05/2020 21:30:15 - INFO - __main__ -     step:252 loss 0.081\n",
      "05/05/2020 21:30:16 - INFO - __main__ -     step:253 loss 0.078\n",
      "05/05/2020 21:30:18 - INFO - __main__ -     step:254 loss 0.095\n",
      "05/05/2020 21:30:19 - INFO - __main__ -     step:255 loss 0.110\n",
      "05/05/2020 21:30:20 - INFO - __main__ -     step:256 loss 0.075\n",
      "05/05/2020 21:30:22 - INFO - __main__ -     step:257 loss 0.190\n",
      "05/05/2020 21:30:23 - INFO - __main__ -     step:258 loss 0.073\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:30:24 - INFO - __main__ -     step:259 loss 0.081\n",
      "05/05/2020 21:30:25 - INFO - __main__ -     step:260 loss 0.057\n",
      "05/05/2020 21:30:27 - INFO - __main__ -     step:261 loss 0.103\n",
      "05/05/2020 21:30:28 - INFO - __main__ -     step:262 loss 0.109\n",
      "05/05/2020 21:30:29 - INFO - __main__ -     step:263 loss 0.092\n",
      "05/05/2020 21:30:31 - INFO - __main__ -     step:264 loss 0.114\n",
      "05/05/2020 21:30:32 - INFO - __main__ -     step:265 loss 0.116\n",
      "05/05/2020 21:30:33 - INFO - __main__ -     step:266 loss 0.094\n",
      "05/05/2020 21:30:34 - INFO - __main__ -     step:267 loss 0.095\n",
      "05/05/2020 21:30:36 - INFO - __main__ -     step:268 loss 0.117\n",
      "05/05/2020 21:30:37 - INFO - __main__ -     step:269 loss 0.081\n",
      "05/05/2020 21:30:38 - INFO - __main__ -     step:270 loss 0.150\n",
      "05/05/2020 21:30:39 - INFO - __main__ -     step:271 loss 0.096\n",
      "05/05/2020 21:30:41 - INFO - __main__ -     step:272 loss 0.111\n",
      "05/05/2020 21:30:42 - INFO - __main__ -     step:273 loss 0.102\n",
      "05/05/2020 21:30:43 - INFO - __main__ -     step:274 loss 0.085\n",
      "05/05/2020 21:30:45 - INFO - __main__ -     step:275 loss 0.092\n",
      "05/05/2020 21:30:46 - INFO - __main__ -     step:276 loss 0.042\n",
      "05/05/2020 21:30:47 - INFO - __main__ -     step:277 loss 0.093\n",
      "05/05/2020 21:30:48 - INFO - __main__ -     step:278 loss 0.113\n",
      "05/05/2020 21:30:49 - INFO - __main__ -     step:279 loss 0.075\n",
      "05/05/2020 21:30:51 - INFO - __main__ -     step:280 loss 0.062\n",
      "05/05/2020 21:30:52 - INFO - __main__ -     step:281 loss 0.137\n",
      "05/05/2020 21:30:53 - INFO - __main__ -     step:282 loss 0.090\n",
      "05/05/2020 21:30:55 - INFO - __main__ -     step:283 loss 0.112\n",
      "05/05/2020 21:30:56 - INFO - __main__ -     step:284 loss 0.097\n",
      "05/05/2020 21:30:57 - INFO - __main__ -     step:285 loss 0.100\n",
      "05/05/2020 21:30:58 - INFO - __main__ -     step:286 loss 0.082\n",
      "05/05/2020 21:31:00 - INFO - __main__ -     step:287 loss 0.066\n",
      "05/05/2020 21:31:01 - INFO - __main__ -     step:288 loss 0.113\n",
      "05/05/2020 21:31:02 - INFO - __main__ -     step:289 loss 0.113\n",
      "05/05/2020 21:31:04 - INFO - __main__ -     step:290 loss 0.051\n",
      "05/05/2020 21:31:05 - INFO - __main__ -     step:291 loss 0.121\n",
      "05/05/2020 21:31:06 - INFO - __main__ -     step:292 loss 0.107\n",
      "05/05/2020 21:31:08 - INFO - __main__ -     step:293 loss 0.089\n",
      "05/05/2020 21:31:09 - INFO - __main__ -     step:294 loss 0.086\n",
      "05/05/2020 21:31:10 - INFO - __main__ -     step:295 loss 0.090\n",
      "05/05/2020 21:31:11 - INFO - __main__ -     step:296 loss 0.106\n",
      "05/05/2020 21:31:13 - INFO - __main__ -     step:297 loss 0.085\n",
      "05/05/2020 21:31:14 - INFO - __main__ -     step:298 loss 0.088\n",
      "05/05/2020 21:31:15 - INFO - __main__ -     step:299 loss 0.065\n",
      "05/05/2020 21:31:17 - INFO - __main__ -     step:300 loss 0.146\n",
      "05/05/2020 21:31:18 - INFO - __main__ -     step:301 loss 0.092\n",
      "05/05/2020 21:31:19 - INFO - __main__ -     step:302 loss 0.074\n",
      "05/05/2020 21:31:20 - INFO - __main__ -     step:303 loss 0.106\n",
      "05/05/2020 21:31:22 - INFO - __main__ -     step:304 loss 0.134\n",
      "05/05/2020 21:31:23 - INFO - __main__ -     step:305 loss 0.078\n",
      "05/05/2020 21:31:24 - INFO - __main__ -     step:306 loss 0.113\n",
      "05/05/2020 21:31:25 - INFO - __main__ -     step:307 loss 0.084\n",
      "05/05/2020 21:31:27 - INFO - __main__ -     step:308 loss 0.079\n",
      "05/05/2020 21:31:28 - INFO - __main__ -     step:309 loss 0.093\n",
      "05/05/2020 21:31:29 - INFO - __main__ -     step:310 loss 0.097\n",
      "05/05/2020 21:31:31 - INFO - __main__ -     step:311 loss 0.116\n",
      "05/05/2020 21:31:31 - INFO - __main__ -     step:312 loss 0.061\n",
      "05/05/2020 21:31:31 - INFO - __main__ -   ***** Running evaluation  *****\n",
      "05/05/2020 21:31:31 - INFO - __main__ -     Num examples = 1000\n",
      "05/05/2020 21:31:31 - INFO - __main__ -     Batch size = 32\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67a80a5beac040e7beb719af21f998e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Evaluating', max=32, style=ProgressStyle(description_width='i…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:31:42 - INFO - __main__ -   ***** Eval results  *****\n",
      "05/05/2020 21:31:42 - INFO - __main__ -     acc_class = [0.974 0.993 1.    0.987 0.926 0.985 0.985 0.998 0.989 0.966 0.937 0.988\n",
      " 0.995 0.921 0.989 0.998 0.945 0.981 0.941 1.    0.941 0.991 0.967 0.965\n",
      " 0.997 0.975 0.987 0.994 0.967 0.975 0.974 1.    0.999 1.    0.952 0.942\n",
      " 0.98  0.994 0.961 0.992 1.    0.989 0.965 0.987 1.    0.941 0.947 0.994\n",
      " 0.982 0.981 1.    0.966 0.994 0.926 0.991]\n",
      "05/05/2020 21:31:42 - INFO - __main__ -     acc_sample = 0.362\n",
      "05/05/2020 21:31:42 - INFO - __main__ -     precision_sample = 0.5518883198062633\n",
      "05/05/2020 21:31:42 - INFO - __main__ -     recall_sample = 0.9592096876991715\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "05/05/2020 21:31:46 - INFO - __main__ -   Saving model checkpoint to ./output_test/checkpoint-939\n",
      "05/05/2020 21:31:54 - INFO - __main__ -   Saving optimizer and scheduler states to ./output_test/checkpoint-939\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "421.96799063682556\n",
      "\n"
     ]
    }
   ],
   "source": [
    "set_seed(args)\n",
    "model.to(args.device)\n",
    "global_step, tr_loss = train(args,cls_train_dataset,cls_val_dataset, model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#用kg的sequence过DGCNN+pooling成固定hidden当作condition layer norm 3 epoch\n",
    "# 05/05/2020 21:31:42 - INFO - __main__ -     acc_sample = 0.362\n",
    "# 05/05/2020 21:31:42 - INFO - __main__ -     precision_sample = 0.5518883198062633\n",
    "# 05/05/2020 21:31:42 - INFO - __main__ -     recall_sample = 0.9592096876991715"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接用kg的sequence当作condition layer norm 3 epoch\n",
    "# 05/05/2020 20:52:09 - INFO - __main__ -     acc_sample = 0.382\n",
    "# 05/05/2020 20:52:09 - INFO - __main__ -     precision_sample = 0.5671302869009827\n",
    "# 05/05/2020 20:52:09 - INFO - __main__ -     recall_sample = 0.9611217335882728"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#单独加kg输入DGCNN输出拼接 3epoch\n",
    "# 05/05/2020 19:03:02 - INFO - __main__ -     acc_sample = 0.338\n",
    "# 05/05/2020 19:03:02 - INFO - __main__ -     precision_sample = 0.5452903120569065\n",
    "# 05/05/2020 19:03:02 - INFO - __main__ -     recall_sample = 0.9630337794773741"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#不加kg 3epoch\n",
    "# 05/05/2020 17:35:24 - INFO - __main__ -     acc_sample = 0.31\n",
    "# 05/05/2020 17:35:24 - INFO - __main__ -     precision_sample = 0.5227034583350231\n",
    "# 05/05/2020 17:35:24 - INFO - __main__ -     recall_sample = 0.9611217335882728"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 直接pooled_output 3 epoch\n",
    "# 04/30/2020 20:23:14 - INFO - __main__ -     acc_sample = 0.369\n",
    "# 04/30/2020 20:23:14 - INFO - __main__ -     precision_sample = 0.5384322080907958\n",
    "# 04/30/2020 20:23:14 - INFO - __main__ -     recall_sample = 0.9588671611598112"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cat pooled_output 3 epoch\n",
    "# 04/30/2020 20:07:12 - INFO - __main__ -     acc_sample = 0.332\n",
    "# 04/30/2020 20:07:12 - INFO - __main__ -     precision_sample = 0.5179671441135593\n",
    "# 04/30/2020 20:07:12 - INFO - __main__ -     recall_sample = 0.9622387053270398"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 原始 3 epoch\n",
    "# 04/30/2020 12:37:43 - INFO - __main__ -     acc_sample = 0.138\n",
    "# 04/30/2020 12:37:43 - INFO - __main__ -     precision_sample = 0.37590158271397733\n",
    "# 04/30/2020 12:37:43 - INFO - __main__ -     recall_sample = 0.9487525286581254"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "#单独使用词向量 3 epoch\n",
    "# 04/30/2020 18:20:01 - INFO - __main__ -     acc_sample = 0.152\n",
    "# 04/30/2020 18:20:01 - INFO - __main__ -     precision_sample = 0.37252301914847574\n",
    "# 04/30/2020 18:20:01 - INFO - __main__ -     recall_sample = 0.9507754551584626"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#拼接 3 epoch\n",
    "# 04/30/2020 11:49:50 - INFO - __main__ -     acc_sample = 0.174\n",
    "# 04/30/2020 11:49:50 - INFO - __main__ -     precision_sample = 0.4019578456787084\n",
    "# 04/30/2020 11:49:50 - INFO - __main__ -     recall_sample = 0.9413351314902225"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "#加上 3 epoch\n",
    "# 04/30/2020 12:15:39 - INFO - __main__ -     acc_sample = 0.153\n",
    "# 04/30/2020 12:15:39 - INFO - __main__ -     precision_sample = 0.37432857832207883\n",
    "# 04/30/2020 12:15:39 - INFO - __main__ -     recall_sample = 0.9399865138233311"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "epoch_iterator=Data.DataLoader(cls_val_dataset,batch_size=2,shuffle=False)\n",
    "for step, batch in enumerate(epoch_iterator):\n",
    "    model.train()\n",
    "    tokenvec=batch[6].detach().cpu().numpy()\n",
    "    batch[6]=torch.tensor(word2vec[tokenvec]).float()\n",
    "    wordvec=batch[7].detach().cpu().numpy()\n",
    "    batch[7]=torch.tensor(word2vec[wordvec]).float()\n",
    "#     batch = tuple(t.to(args.device) for t in batch[:-1])\n",
    "    inputs = {\"input_ids\": batch[0], \"attention_mask\": batch[1], \"rel_label\": batch[3],\\\n",
    "             \"postag\":batch[4],\"feature\":batch[5],\"tokenvec\":batch[6],\"wordvec\":batch[7],\"wordmask\":batch[8],\"plan_label\":batch[9].float()}\n",
    "    inputs[\"token_type_ids\"]=batch[2]\n",
    "    outputs = model(**inputs)\n",
    "    outputs[0].backward()\n",
    "    break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fjw",
   "language": "python",
   "name": "fjw"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
